def _testConfMatrixOnTensors(self, tf_dtype, np_dtype): with self.test_session() as sess: m_neg = tf.placeholder(dtype=tf.float32) m_pos = tf.placeholder(dtype=tf.float32) s = tf.placeholder(dtype=tf.float32) neg = tf.random_normal([20], mean=m_neg, stddev=s, dtype=tf.float32) pos = tf.random_normal([20], mean=m_pos, stddev=s, dtype=tf.float32) data = tf.concat_v2([neg, pos], 0) data = tf.cast(tf.round(data), tf_dtype) data = tf.minimum(tf.maximum(data, 0), 1) lab = tf.concat_v2( [tf.zeros( [20], dtype=tf_dtype), tf.ones( [20], dtype=tf_dtype)], 0) cm = tf.confusion_matrix( lab, data, dtype=tf_dtype, num_classes=2) d, l, cm_out = sess.run([data, lab, cm], {m_neg: 0.0, m_pos: 1.0, s: 1.0}) truth = np.zeros([2, 2], dtype=np_dtype) try: range_builder = xrange except NameError: # In Python 3. range_builder = range for i in range_builder(len(d)): truth[d[i], l[i]] += 1 self.assertEqual(cm_out.dtype, np_dtype) self.assertAllClose(cm_out, truth, atol=1e-10)
def _testConfMatrix(self, predictions, labels, truth, weights=None): with self.test_session(): dtype = predictions.dtype ans = tf.confusion_matrix( labels, predictions, dtype=dtype, weights=weights) tf_ans = ans.eval() self.assertAllClose(tf_ans, truth, atol=1e-10) self.assertEqual(tf_ans.dtype, dtype)
def testOutputIsInt64(self): predictions = np.arange(2) labels = np.arange(2) with self.test_session(): cm = tf.confusion_matrix( labels, predictions, dtype=dtypes.int64) tf_cm = cm.eval() self.assertEqual(tf_cm.dtype, np.int64)
def train(sess, mnist, n_training_epochs, batch_size, summaries_op, accuracy_summary_op, train_writer, test_writer, X, Y, train_op, loss_op, accuracy_op): # compute number of batches for given batch_size num_train_batches = mnist.train.num_examples // batch_size # record starting time train_start = time.time() # Run through the entire dataset n_training_epochs times for i in range(n_training_epochs): # Initialise statistics training_loss = 0 epoch_start = time.time() # Run the SGD train op for each minibatch for _ in range(num_train_batches): batch = mnist.train.next_batch(batch_size) trainstep_result, batch_loss, summary = \ qfns.train_step(sess, batch, X, Y, train_op, loss_op, summaries_op) train_writer.add_summary(summary, i) training_loss += batch_loss # Timing and statistics epoch_duration = round(time.time() - epoch_start, 2) ave_train_loss = training_loss / num_train_batches # Get accuracy train_accuracy = \ accuracy(sess, mnist.train, batch_size, X, Y, accuracy_op) test_accuracy = \ accuracy(sess, mnist.test, batch_size, X, Y, accuracy_op) # log accuracy at the current epoch on training and test sets train_acc_summary = sess.run(accuracy_summary_op, feed_dict={accuracy_placeholder: train_accuracy}) train_writer.add_summary(train_acc_summary, i) test_acc_summary = sess.run(accuracy_summary_op, feed_dict={accuracy_placeholder: test_accuracy}) test_writer.add_summary(test_acc_summary, i) [writer.flush() for writer in [train_writer, test_writer]] train_duration = round(time.time() - train_start, 2) # Output to montior training print('Epoch {0}, Training Loss: {1}, Test accuracy: {2}, \ time: {3}s, total time: {4}s'.format(i, ave_train_loss, test_accuracy, epoch_duration, train_duration)) print('Total training time: {0}s'.format(train_duration)) print('Confusion Matrix:') true_class=tf.argmax(Y, 1) predicted_class=tf.argmax(preds_op, 1) cm=tf.confusion_matrix(predicted_class,true_class) print(sess.run(cm, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))
def init_training_graph(self): with tf.name_scope('Evaluation'): self.logits = self.conv_layer_f(self.last, self.logits_weight, strides=[1,1,1,1], scope_name="logits/") self.predictions = tf.argmax(self.logits, axis=3) with tf.name_scope('Loss'): self.loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=tf.squeeze(tf.cast(self.train_labels_node, tf.int32), squeeze_dims=[3]), name="entropy"))) tf.summary.scalar("entropy", self.loss) with tf.name_scope('Accuracy'): LabelInt = tf.squeeze(tf.cast(self.train_labels_node, tf.int64), squeeze_dims=[3]) CorrectPrediction = tf.equal(self.predictions, LabelInt) self.accuracy = tf.reduce_mean(tf.cast(CorrectPrediction, tf.float32)) tf.summary.scalar("accuracy", self.accuracy) with tf.name_scope('ClassPrediction'): flat_LabelInt = tf.reshape(LabelInt, [-1]) flat_predictions = tf.reshape(self.predictions, [-1]) self.cm = tf.confusion_matrix(flat_LabelInt, flat_predictions, self.NUM_LABELS) flatten_confusion_matrix = tf.reshape(self.cm, [-1]) total = tf.reduce_sum(self.cm) for i in range(self.NUM_LABELS): name = "Label_{}".format(i) TP, TN, FP, FN = GetCMInfo_TF(self.cm, i, self.NUM_LABELS) precision = tf.divide(TP, tf.add(TP, FP)) recall = tf.divide(TP, tf.add(TP, FN)) num = tf.multiply(precision, recall) dem = tf.add(precision, recall) F1 = tf.scalar_mul(2, tf.divide(num, dem)) Nprecision = tf.divide(TN, tf.add(TN, FN)) MeanAcc = tf.divide(tf.add(precision, Nprecision) ,2) tf.summary.scalar(name + '_Precision', precision) tf.summary.scalar(name + '_Recall', recall) tf.summary.scalar(name + '_F1', F1) tf.summary.scalar(name + '_Performance', MeanAcc) confusion_image = tf.reshape( tf.cast( self.cm, tf.float32), [1, self.NUM_LABELS, self.NUM_LABELS, 1]) tf.summary.image('confusion', confusion_image) self.train_prediction = tf.nn.softmax(self.logits) self.test_prediction = self.train_prediction tf.global_variables_initializer().run() print('Computational graph initialised')
def eval_confusion_matrix(labels, predictions, params): with tf.variable_scope("eval_confusion_matrix"): con_matrix = tf.confusion_matrix(labels=labels, predictions=predictions, num_classes=params['num_classes']) con_matrix_sum = tf.Variable( tf.zeros( shape=(params['num_classes'], params['num_classes']), dtype=tf.int32 ), trainable=False, name="confusion_matrix_result", collections=[tf.GraphKeys.LOCAL_VARIABLES] ) update_op = tf.assign_add(con_matrix_sum, con_matrix) return tf.convert_to_tensor(con_matrix_sum), update_op
def main(_): # We want to see all the logging messages for this tutorial. tf.logging.set_verbosity(tf.logging.INFO) # Start a new TensorFlow session. sess = tf.InteractiveSession() # Begin by making sure we have the training data we need. If you already have # training data of your own, use `--data_url= ` on the command line to avoid # downloading. model_settings = models.prepare_model_settings( len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))), FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms, FLAGS.window_stride_ms, FLAGS.dct_coefficient_count) audio_processor = input_data.AudioProcessor( FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage, FLAGS.wanted_words.split(','), FLAGS.validation_percentage, FLAGS.testing_percentage, model_settings) fingerprint_size = model_settings['fingerprint_size'] label_count = model_settings['label_count'] time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000) # Figure out the learning rates for each training phase. Since it's often # effective to have high learning rates at the start of training, followed by # lower levels towards the end, the number of steps and learning rates can be # specified as comma-separated lists to define the rate at each stage. For # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001 # will run 13,000 training loops in total, with a rate of 0.001 for the first # 10,000, and 0.0001 for the final 3,000. training_steps_list = list(map(int, FLAGS.how_many_training_steps.split(','))) learning_rates_list = list(map(float, FLAGS.learning_rate.split(','))) if len(training_steps_list) != len(learning_rates_list): raise Exception( '--how_many_training_steps and --learning_rate must be equal length ' 'lists, but are %d and %d long instead' % (len(training_steps_list), len(learning_rates_list))) fingerprint_input = tf.placeholder( tf.float32, [None, fingerprint_size], name='fingerprint_input') print 'fingerprint_input',fingerprint_input logits, dropout_prob = models.create_model( fingerprint_input, model_settings, FLAGS.model_architecture, is_training=True) # Define loss and optimizer ground_truth_input = tf.placeholder( tf.float32, [None, label_count], name='groundtruth_input') # Optionally we can add runtime checks to spot when NaNs or other symptoms of # numerical errors start occurring during training. control_dependencies = [] if FLAGS.check_nans: checks = tf.add_check_numerics_ops() control_dependencies = [checks] # Create the back propagation and training evaluation machinery in the graph. with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits)) tf.summary.scalar('cross_entropy', cross_entropy_mean) with tf.name_scope('train'), tf.control_dependencies(control_dependencies): learning_rate_input = tf.placeholder( tf.float32, [], name='learning_rate_input') train_step = tf.train.GradientDescentOptimizer( learning_rate_input).minimize(cross_entropy_mean) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', evaluation_step) global_step = tf.contrib.framework.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) saver = tf.train.Saver(tf.global_variables()) # Merge all the summaries and write them out to /tmp/retrain_logs (by default) merged_summaries = tf.summary.merge_all() train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation') tf.global_variables_initializer().run() start_step = 1 if FLAGS.start_checkpoint: models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) start_step = global_step.eval(session=sess) tf.logging.info('Training from step: %d ', start_step) # Save graph.pbtxt. tf.train.write_graph(sess.graph_def, FLAGS.train_dir, FLAGS.model_architecture + '.pbtxt') # Save list of words. with gfile.GFile( os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '_labels.txt'), 'w') as f: f.write('\n'.join(audio_processor.words_list)) # Training loop. training_steps_max = np.sum(training_steps_list) for training_step in xrange(start_step, training_steps_max + 1): # Figure out what the current learning rate is. training_steps_sum = 0 for i in range(len(training_steps_list)): training_steps_sum += training_steps_list[i] if training_step <= training_steps_sum: learning_rate_value = learning_rates_list[i] break # Pull the audio samples we'll use for training. train_fingerprints, train_ground_truth = audio_processor.get_data( FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency, FLAGS.background_volume, time_shift_samples, 'training', sess) # Run the graph with this batch of training data. train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run( [ merged_summaries, evaluation_step, cross_entropy_mean, train_step, increment_global_step ], feed_dict={ fingerprint_input: train_fingerprints, ground_truth_input: train_ground_truth, learning_rate_input: learning_rate_value, dropout_prob: 0.5 }) train_writer.add_summary(train_summary, training_step) tf.logging.info('Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' % (training_step, learning_rate_value, train_accuracy * 100, cross_entropy_value)) is_last_step = (training_step == training_steps_max) if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step: set_size = audio_processor.set_size('validation') total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): validation_fingerprints, validation_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'validation', sess)) # Run a validation step and capture training summaries for TensorBoard # with the `merged` op. validation_summary, validation_accuracy, conf_matrix = sess.run( [merged_summaries, evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: validation_fingerprints, ground_truth_input: validation_ground_truth, dropout_prob: 1.0 }) validation_writer.add_summary(validation_summary, training_step) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (validation_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' % (training_step, total_accuracy * 100, set_size)) # Save the model checkpoint periodically. if (training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max): checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '.ckpt') tf.logging.info('Saving to "%s-%d"', checkpoint_path, training_step) saver.save(sess, checkpoint_path, global_step=training_step) set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): test_fingerprints, test_ground_truth = audio_processor.get_data( FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess) test_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, dropout_prob: 1.0 }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (test_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Final test accuracy = %.1f%% (N=%d)' % (total_accuracy * 100, set_size))
def main(_): NUM_INPUTS = FLAGS.input_size NUM_HISTORY = FLAGS.history_size # amount of history we are keeping of previous measurements NUM_CLASSES = FLAGS.output_size # the data, split between train and test sets x_train, y_train, x_test, y_test = generate_simulated_data(10000, balance=False) # append more data to x_train and x_test for _ in range(NUM_HISTORY - 1): a, _, at, _ = generate_simulated_data(10000, balance=False) x_train = np.concatenate([x_train, a], axis=1) x_test = np.concatenate([x_test, at], axis=1) x_train = x_train.astype('uint8') x_test = x_test.astype('uint8') print(x_train.shape, 'train samples') print(x_test.shape, 'test samples') # convert class vectors to binary class matrices y_train = y_train.astype('int64') y_test = y_test.astype('int64') tf.logging.set_verbosity(tf.logging.INFO) sess = tf.InteractiveSession() # Figure out the learning rates for each training phase. Since it's often # effective to have high learning rates at the start of training, followed by # lower levels towards the end, the number of steps and learning rates can be # specified as comma-separated lists to define the rate at each stage. For # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001 # will run 13,000 training loops in total, with a rate of 0.001 for the first # 10,000, and 0.0001 for the final 3,000. training_steps_list = list( map(int, FLAGS.how_many_training_steps.split(','))) learning_rates_list = list(map(float, FLAGS.learning_rate.split(','))) if len(training_steps_list) != len(learning_rates_list): raise Exception( '--how_many_training_steps and --learning_rate must be equal length ' 'lists, but are %d and %d long instead' % (len(training_steps_list), len(learning_rates_list))) input_placeholder = tf.placeholder(tf.float32, [None, NUM_INPUTS * NUM_HISTORY], name='graph_input') if FLAGS.quantize: input_min, input_max = 0, 256 graph_input = tf.fake_quant_with_min_max_args(input_placeholder, input_min, input_max) else: graph_input = input_placeholder logits, dropout_prob = models.create_conv_model(graph_input, NUM_INPUTS, NUM_HISTORY, NUM_CLASSES, is_training=True) # Define loss and optimizer ground_truth_input = tf.placeholder(tf.int64, [None], name='groundtruth_input') # Optionally we can add runtime checks to spot when NaNs or other symptoms of # numerical errors start occurring during training. control_dependencies = [] if FLAGS.check_nans: checks = tf.add_check_numerics_ops() control_dependencies = [checks] # Create the back propagation and training evaluation machinery in the graph. with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy( labels=ground_truth_input, logits=logits) if FLAGS.quantize: tf.contrib.quantize.create_training_graph(quant_delay=0) with tf.name_scope('train'), tf.control_dependencies(control_dependencies): learning_rate_input = tf.placeholder(tf.float32, [], name='learning_rate_input') train_step = tf.train.GradientDescentOptimizer( learning_rate_input).minimize(cross_entropy_mean) predicted_indices = tf.argmax(logits, 1) correct_prediction = tf.equal(predicted_indices, ground_truth_input) confusion_matrix = tf.confusion_matrix(ground_truth_input, predicted_indices, num_classes=NUM_CLASSES) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) with tf.get_default_graph().name_scope('eval'): tf.summary.scalar('cross_entropy', cross_entropy_mean) tf.summary.scalar('accuracy', evaluation_step) global_step = tf.train.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) saver = tf.train.Saver(tf.global_variables()) # Merge all the summaries and write them out to /tmp/retrain_logs (by default) merged_summaries = tf.summary.merge_all(scope='eval') train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation') tf.global_variables_initializer().run() start_step = 1 tf.logging.info('Training from step: %d ', start_step) # Save graph.pbtxt. tf.train.write_graph(sess.graph_def, FLAGS.train_dir, FLAGS.model_architecture + '.pbtxt') # Training loop. training_steps_max = np.sum(training_steps_list) for training_step in xrange(start_step, training_steps_max + 1): # Figure out what the current learning rate is. training_steps_sum = 0 for i in range(len(training_steps_list)): training_steps_sum += training_steps_list[i] if training_step <= training_steps_sum: learning_rate_value = learning_rates_list[i] break # Pull the audio samples we'll use for training. index = (training_step * FLAGS.batch_size) % x_train.shape[0] train_fingerprints = x_train[index:index + FLAGS.batch_size] train_ground_truth = y_train[index:index + FLAGS.batch_size] # Run the graph with this batch of training data. train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run( [ merged_summaries, evaluation_step, cross_entropy_mean, train_step, increment_global_step, ], feed_dict={ graph_input: train_fingerprints, ground_truth_input: train_ground_truth, learning_rate_input: learning_rate_value, dropout_prob: 0.5 }) train_writer.add_summary(train_summary, training_step) tf.logging.info( 'Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' % (training_step, learning_rate_value, train_accuracy * 100, cross_entropy_value)) is_last_step = (training_step == training_steps_max) if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step: set_size = y_test.shape[0] total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): validation_fingerprints = x_test[i:i + FLAGS.batch_size] validation_ground_truth = y_test[i:i + FLAGS.batch_size] # Run a validation step and capture training summaries for TensorBoard # with the `merged` op. validation_summary, validation_accuracy, conf_matrix = sess.run( [merged_summaries, evaluation_step, confusion_matrix], feed_dict={ graph_input: validation_fingerprints, ground_truth_input: validation_ground_truth, dropout_prob: 1.0 }) validation_writer.add_summary(validation_summary, training_step) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (validation_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' % (training_step, total_accuracy * 100, set_size)) # Save the model checkpoint periodically. if (training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max): checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '.ckpt') tf.logging.info('Saving to "%s-%d"', checkpoint_path, training_step) saver.save(sess, checkpoint_path, global_step=training_step)
def confmat(logits, labels): preds = tf.argmax(logits, axis=1) return tf.confusion_matrix(labels, preds)
def main(): print('Starting training.') x, y, num_examples = tfrecords_loader(n_classes, batch_size, hm_epochs) global_step = tf.Variable(0, name='global_step', trainable=False) with tf.Session() as sess: prediction = convolutional_neural_network(x) weights_fc = [ v for v in tf.global_variables() if v.name == "fully_connected1/weights/fully_connected1weights:0" ][0] weights_fl = [ v for v in tf.global_variables() if v.name == "output/output_weights/outputweights:0" ][0] print(weights_fc.shape) print(weights_fl.shape) # defining name scopes for cost and accuracy to be written into the summary file with tf.name_scope('training'): with tf.name_scope('cross_entropy'): cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=prediction)) regularizer_fc = tf.nn.l2_loss(weights_fc) regularizer_fl = tf.nn.l2_loss(weights_fl) cost = tf.reduce_mean(cost + beta * (regularizer_fc + regularizer_fl)) tf.summary.scalar('cross_entropy', cost) with tf.name_scope('train'): optimizer = tf.train.AdamOptimizer().minimize( cost, global_step=global_step ) # using Adam Optimizer algorithm for reducing cost with tf.name_scope('accuracy'): with tf.name_scope('correct_prediction'): correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1)) with tf.name_scope('train_accuracy'): accuracy = tf.reduce_mean(tf.cast(correct, 'float')) tf.summary.scalar('train_accuracy', accuracy) if os.path.isfile(current_location + "/saves/model.ckpt.meta"): saver = tf.train.Saver() saver.restore(sess, current_location + "/saves/model.ckpt") print("Model restored.") else: saver = tf.train.Saver() print("Previous save missing...\nStarting with random values.") tf.global_variables_initializer().run() tf.local_variables_initializer().run( ) # loads images into x and y variables coord = tf.train.Coordinator( ) # coordinator used for coordinating between various threads that load data threads = tf.train.start_queue_runners(sess=sess, coord=coord) # bitno merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter(current_location, sess.graph) n = int(num_examples / batch_size) - 1 for epoch in range(hm_epochs): epoch_loss = 0 # run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) # run_metadata = tf.RunMetadata() for i in tqdm(range(n)): if i % 5 == 0: # summary, _, c = sess.run([merged, optimizer, cost], options=run_options, # run_metadata=run_metadata) summary, _, c, step = sess.run( [merged, optimizer, cost, global_step]) train_writer.add_summary(summary, (epoch * n + i) / 5 + 641) train_writer.add_session_log( tf.SessionLog(status=tf.SessionLog.START), global_step=step) # Create the Timeline object, and write it to a json # tl = timeline.Timeline(run_metadata.step_stats) # ctf = tl.generate_chrome_trace_format() # with open('timeline' + str(i) + '.json', 'w') as f: # f.write(ctf) if i % 75 == 0 and i > 0: new_path = current_location + "/saves" if not os.path.exists(new_path): os.makedirs(new_path) saver.save(sess, new_path + "/model.ckpt") else: _, c = sess.run([optimizer, cost]) epoch_loss += c # Creating saves after an epoch ts = time.time() st = datetime.datetime.fromtimestamp(ts).strftime('%m-%d-%H-%M') new_path = current_location + "/saves" if not os.path.exists(new_path): os.makedirs(new_path) save_path = saver.save(sess, new_path + "/model" + st + ".ckpt") saver.save(sess, new_path + "/model.ckpt") print("Model saved in file: %s" % save_path) print('Epoch', epoch, 'completed out of', hm_epochs, 'loss:', epoch_loss) # Added output of current Confusion Matrix conf_matrix = tf.confusion_matrix(tf.argmax(y, 1), tf.argmax(prediction, 1), num_classes=n_classes) conf_matrix_eval, test_accuracy = sess.run([conf_matrix, accuracy]) with open(current_location + "/confMatrix{}.txt".format(epoch), 'w') as confMatrixOutput: for line in conf_matrix_eval: for word in line: confMatrixOutput.write('{:>4}'.format(word)) confMatrixOutput.write('\n') # Print out the current test accuracy print('Accuracy:', test_accuracy) coord.request_stop() coord.join(threads)
def get_graph(model_settings, is_last_model=False): # Build a graph containing `net1`. with tf.Graph().as_default() as net1_graph: fingerprint_size = model_settings['fingerprint_size'] label_count = model_settings['label_count'] fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size], name='fingerprint_input') logits, dropout_prob = models.create_model(fingerprint_input, model_settings, FLAGS.model_architecture, is_training=True) # Define loss and optimizer ground_truth_input = tf.placeholder(tf.float32, [None, label_count], name='groundtruth_input') ws_input = tf.placeholder(tf.float32, [ None, ], name='input_w') # Optionally we can add runtime checks to spot when NaNs or other symptoms of # numerical errors start occurring during training. control_dependencies = [] if FLAGS.check_nans: checks = tf.add_check_numerics_ops() control_dependencies = [checks] # Create the back propagation and training evaluation machinery in the graph. with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits)) cross_entropy_mean_w = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits) * ws_input) tf.summary.scalar('cross_entropy', cross_entropy_mean) with tf.name_scope('train'), tf.control_dependencies( control_dependencies): learning_rate_input = tf.placeholder(tf.float32, [], name='learning_rate_input') # train_step = tf.train.GradientDescentOptimizer( # learning_rate_input).minimize(cross_entropy_mean) train_step = tf.train.AdamOptimizer( learning_rate_input, epsilon=1e-6).minimize(cross_entropy_mean_w) prob = tf.nn.softmax(logits) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices) evaluation_step = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', evaluation_step) global_step = tf.contrib.framework.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) saver = tf.train.Saver(tf.global_variables()) init_op = tf.global_variables_initializer() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.5 # sess = tf.Session(config=config) if is_last_model: sess = tf.Session(graph=net1_graph, config=config) else: sess = tf.InteractiveSession(graph=net1_graph, config=config) sess.run(init_op) # saver.restore(sess, 'epoch_10.ckpt') if is_last_model: if FLAGS.start_checkpoint: # models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) saver.restore(sess, FLAGS.start_checkpoint) return sess, saver, logits, dropout_prob, fingerprint_input, ground_truth_input,ws_input, learning_rate_input,\ increment_global_step,confusion_matrix, evaluation_step, prob, predicted_indices,cross_entropy_mean,\ train_step
for frame in features: x_test.append(list(flatten(frame.tolist()))) video_correct, predict = sess.run([correct, y_video], feed_dict={x: x_test, y_: y_test}) Predicts.append(predict) numVideo += 1 if(video_correct): print('\t' + ea_video) numCorrect += 1 print('\tAccuracy: %s' %(numCorrect/numVideo)) res.write(ea_video + '\tTruth:%s\tPredict:%s\n' %(y_test, predict)) y_p = tf.placeholder(tf.float32, [None]) y_t = tf.placeholder(tf.float32, [None]) cMatrix = tf.confusion_matrix(y_p, y_t) cMatrix_res = sess.run(cMatrix, feed_dict={y_p: Truth, y_t: Predicts}) print(cMatrix_res.shape) np.save(nn_dir + 'confusion_matrix_0621_perVideo.npy', cMatrix_res) #for x in cMatrix_res: # print(x) res.close() #print(cMatrix_res.shape) #np.save(result_dir + 'confusion_matrix_0618.npy', cMatrix_res) #print(cMatrix_res) #test_accuracy = accuracy.eval(feed_dict={x: x_test, y_: y_test}) #print("test accuracy %g"%train_accuracy)
def construct(self, args): with self.session.graph.as_default(): # Inputs self.images = tf.placeholder(tf.float32, [None, self.WIDTH, self.HEIGHT, 1], name="images") self.labels = tf.placeholder(tf.int64, [None], name="labels") # Computation flattened_images = tf.layers.flatten(self.images, name="flatten") # add args.layers hidden layers with activations given by # args.activation and store results in hidden_layer. Possible # activations are none, relu, tanh and sigmoid. if args.activation == 'relu': activation = tf.nn.relu elif args.activation == 'tanh': activation = tf.nn.tanh elif args.activation == 'sigmoid': activation = tf.nn.sigmoid else: activation = None hidden_layer = flattened_images for layer_id in range(args.layers): hidden_layer = tf.layers.dense(hidden_layer, args.hidden_layer, activation=activation, name="hidden_layer_%d" % layer_id) output_layer = tf.layers.dense(hidden_layer, self.LABELS, activation=None, name="output_layer") self.predictions = tf.argmax(output_layer, axis=1) # Training loss = tf.losses.sparse_softmax_cross_entropy(self.labels, output_layer, scope="loss") global_step = tf.train.create_global_step() self.training = tf.train.GradientDescentOptimizer(0.03).minimize( loss, global_step=global_step, name="training") # Summaries self.accuracy = tf.reduce_mean( tf.cast(tf.equal(self.labels, self.predictions), tf.float32)) confusion_matrix = tf.reshape( tf.confusion_matrix(self.labels, self.predictions, weights=tf.not_equal( self.labels, self.predictions), dtype=tf.float32), [1, self.LABELS, self.LABELS, 1]) summary_writer = tf.contrib.summary.create_file_writer( args.logdir, flush_millis=10 * 1000) self.summaries = {} with summary_writer.as_default( ), tf.contrib.summary.record_summaries_every_n_global_steps(100): self.summaries["train"] = [ tf.contrib.summary.scalar("train/loss", loss), tf.contrib.summary.scalar("train/accuracy", self.accuracy) ] with summary_writer.as_default( ), tf.contrib.summary.always_record_summaries(): for dataset in ["dev", "test"]: self.summaries[dataset] = [ tf.contrib.summary.scalar(dataset + "/accuracy", self.accuracy), tf.contrib.summary.image(dataset + "/confusion_matrix", confusion_matrix) ] # Initialize variables self.session.run(tf.global_variables_initializer()) with summary_writer.as_default(): tf.contrib.summary.initialize(session=self.session, graph=self.session.graph)
def train_confusion_fn(): train_confusion = tf.confusion_matrix(tf.argmax(y_train, 1), tf.argmax(logit_train, 1)) train_confusion = tf.to_float(train_confusion) / tf.constant((logit_train.shape.as_list()[0] / float(logit_train.shape.as_list()[1])), dtype=tf.float32) train_confusion = tf.expand_dims(tf.expand_dims(train_confusion, 0), 3) return train_confusion
#tf.summary.scalar('loss', cross_entropy) #merged = tf.summary.merge_all() #train_writer = tf.summary.FileWriter(nn_run_dir + '/train', sess.graph) #test_writer = tf.summary.FileWriter(nn_run_dir + '/test', sess.graph) #saver = tf.train.Saver(max_to_keep = 10) #sess.run(tf.global_variables_initializer()) #y_train = sess.run(y_train) #y_test = sess.run(y_test) saver = tf.train.Saver() saver.restore(sess, nn_run_dir + 'model.ckpt') print('model restored') cMatrix = tf.confusion_matrix(y, y_) acc, cMatrix_res = sess.run([accuracy, cMatrix], feed_dict={x: x_test, y_: y_test}) print(acc) print(cMatrix_res.shape) np.save(result_dir + 'confusion_matrix_0621_per_frame.npy', cMatrix_res) #print(cMatrix_res) #test_accuracy = accuracy.eval(feed_dict={x: x_test, y_: y_test}) #print("test accuracy %g"%train_accuracy) #f_test.write("step %d, test accuracy %g\n"%(step-1, test_accuracy)) #f_test.close() #f_train.close() # if current_step % FLAGS.checkpoint_every == 0: # path = saver.save(sess, checkpoint_prefix, global_step=current_step) # print("Saved model checkpoint to {}\n".format(path))
train_writer.add_summary(merged_op, i) test_writer.add_summary(TestMerge, i) step += 1 print("Optimization Finished!") print("save model") save_path = saver.save(sess, "./Test_CNN_model_0509_4096_3_3_CH4_Gul12Exp_v1.ckpt") print("save model:{0} Finished".format(save_path)) # Calculate accuracy TrainTimer = timer() pred_class_index, accuracy = sess.run([pred_class_index, accuracy], feed_dict={ x: TestData, y: TestLabel, keep_prob: 1. }) print(pred_class_index) True_class_index = tf.argmax(TestLabel, 1) TestCon = tf.confusion_matrix(True_class_index, pred_class_index, num_classes=12) print("TestCon:", '\n', sess.run(TestCon)) scio.savemat('TestCon.mat', {'TestCon': sess.run(TestCon)}) scio.savemat('TestPred.mat', {'TestPred': TestPred}) scio.savemat('True_class_index.mat', {'True_class_index': sess.run(True_class_index)}) print("Testing pred:", pred) print("Testing Accuracy:", accuracy)
def train_3d_nn(): time0 = time.time() chunks_ids = get_ids(DATA_PATH) X, Y = get_data(chunks_ids, DATA_PATH) print('X.shape', X.shape) print('Y.shape', Y.shape) print('np.bincount(Y)', np.bincount(Y.astype(dtype=np.int64))) non_zero_nodules = Y != 0.0 X = X[non_zero_nodules] Y = Y[non_zero_nodules] #print(Y_merge_class_3_2) #X = X[Y_merge_class_3_2] #Y = Y[Y_merge_class_3_2] # print('X.shape', X.shape) # print('Y.shape', Y.shape) # print('np.bincount(Y)', np.bincount(Y.astype(dtype=np.int64))) # SAMPLE_SIZE = 60 # x = np.ndarray((SAMPLE_SIZE*6), dtype=np.object) # y = np.ndarray((SAMPLE_SIZE*6), dtype=np.float32) # x[0:SAMPLE_SIZE] = X[Y==0.0][0:SAMPLE_SIZE] # x[SAMPLE_SIZE: SAMPLE_SIZE*2] = X[Y==1.0][0:SAMPLE_SIZE] # x[SAMPLE_SIZE*2:SAMPLE_SIZE*3] = X[Y==2.0][0:SAMPLE_SIZE] # x[SAMPLE_SIZE*3:SAMPLE_SIZE*4] = X[Y==3.0][0:SAMPLE_SIZE] # x[SAMPLE_SIZE*4:SAMPLE_SIZE*5] = X[Y==4.0][0:SAMPLE_SIZE] # x[SAMPLE_SIZE*5:SAMPLE_SIZE*6] = X[Y==5.0][0:SAMPLE_SIZE] # y[0:SAMPLE_SIZE] = [0.0]*SAMPLE_SIZE # y[SAMPLE_SIZE:SAMPLE_SIZE*2] = [1.0]*SAMPLE_SIZE # y[SAMPLE_SIZE*2:SAMPLE_SIZE*3] = [2.0]*SAMPLE_SIZE # y[SAMPLE_SIZE*3:SAMPLE_SIZE*4] = [3.0]*SAMPLE_SIZE # y[SAMPLE_SIZE*4:SAMPLE_SIZE*5] = [4.0]*SAMPLE_SIZE # y[SAMPLE_SIZE*5:SAMPLE_SIZE*6] = [5.0]*SAMPLE_SIZE # print(x) # print(y) # print('Using SAMPLE_SIZE', SAMPLE_SIZE) # print('x.shape', x.shape) # print('y.shape', y.shape) print("Total time to load data: " + str(timedelta(seconds=int(round(time.time() - time0))))) print('Splitting into train, validation sets') train_x, validation_x, train_y, validation_y = model_selection.train_test_split( X, Y, random_state=42, stratify=Y, test_size=0.20) klass_weights = np.asarray([ 1025.0 / 7013.0, 1638.0 / 7013.0, 2673.0 / 7013.0, 968.0 / 7013.0, 709.0 / 7013.0 ]) # Free up X and Y memory del X del Y # del x # del y print("Total time to split: " + str(timedelta(seconds=int(round(time.time() - time0))))) print('train_x: {}'.format(train_x.shape)) print('validation_x: {}'.format(validation_x.shape)) print('train_y: {}'.format(train_y.shape)) print('validation_y: {}'.format(validation_y.shape)) print('np.bincount(train_y)', np.bincount(train_y.astype(dtype=np.int64))) print('np.bincount(validation_y)', np.bincount(validation_y.astype(dtype=np.int64))) train_y = (np.arange(FLAGS.num_classes) == train_y[:, None]) + 0 validation_y = (np.arange(FLAGS.num_classes) == validation_y[:, None]) + 0 # Seed numpy random to generate identical random numbers every time (used in batching) np.random.seed(42) def get_validation_batch(validation_x_ids, validation_y, batch_number): num_images = len(validation_x_ids) count = 0 start_index = batch_number * FLAGS.batch_size end_index = start_index + FLAGS.batch_size end_index = num_images if end_index > num_images else end_index real_batch_size = end_index - start_index validation_x = np.ndarray([ real_batch_size, FLAGS.chunk_size, FLAGS.chunk_size, FLAGS.chunk_size, 1 ], dtype=np.float32) for chunk_id in validation_x_ids[start_index:end_index]: chunk = np.load(DATA_PATH + chunk_id + '_X.npy').astype(np.float32, copy=False) validation_x[count, :, :, :, :] = img_to_rgb(chunk) count = count + 1 return validation_x, validation_y[start_index:end_index] def feed_dict(is_train, batch_number=0): if is_train: x_batch, y_batch = get_batch(train_x, train_y) k = FLAGS.keep_prob else: x_batch, y_batch = get_validation_batch(validation_x, validation_y, batch_number) k = 1.0 crss_entrpy_weights = np.ones((y_batch.shape[0])) for m in range(y_batch.shape[0]): crss_entrpy_weights[m] = np.amax(y_batch[m] * klass_weights) return { x: x_batch, y_labels: y_batch, keep_prob: k, cross_entropy_weights: crss_entrpy_weights } # Graph construction graph = tf.Graph() with graph.as_default(): x = tf.placeholder(tf.float32, shape=[ None, FLAGS.chunk_size, FLAGS.chunk_size, FLAGS.chunk_size, 1 ], name='x') y = tf.placeholder(tf.float32, shape=[None, FLAGS.num_classes], name='y') y_labels = tf.placeholder(tf.float32, shape=[None, FLAGS.num_classes], name='y_labels') cross_entropy_weights = tf.placeholder(tf.float32, shape=[None], name='cross_entropy_weights') keep_prob = tf.placeholder(tf.float32) class_weights_base = tf.ones_like(y_labels) class_weights = tf.multiply(class_weights_base, [ 1025.0 / 7013.0, 1638.0 / 7013.0, 2673.0 / 7013.0, 968.0 / 7013.0, 709.0 / 7013.0 ]) # layer1 conv1_1_out, conv1_1_weights = conv3d(inputs=x, filter_size=3, num_filters=16, num_channels=1, strides=[1, 3, 3, 3, 1], layer_name='conv1_1') relu1_1_out = relu_3d(inputs=conv1_1_out, layer_name='relu1_1') conv1_2_out, conv1_2_weights = conv3d(inputs=relu1_1_out, filter_size=3, num_filters=16, num_channels=16, strides=[1, 3, 3, 3, 1], layer_name='conv1_2') relu1_2_out = relu_3d(inputs=conv1_2_out, layer_name='relu1_2') pool1_out = max_pool_3d(inputs=relu1_2_out, filter_size=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], layer_name='pool1') # layer2 conv2_1_out, conv2_1_weights = conv3d(inputs=pool1_out, filter_size=3, num_filters=64, num_channels=16, strides=[1, 3, 3, 3, 1], layer_name='conv2_1') relu2_1_out = relu_3d(inputs=conv2_1_out, layer_name='relu2_1') conv2_2_out, conv2_2_weights = conv3d(inputs=relu2_1_out, filter_size=3, num_filters=64, num_channels=64, strides=[1, 3, 3, 3, 1], layer_name='conv2_2') relu2_2_out = relu_3d(inputs=conv2_2_out, layer_name='relu2_2') pool2_out = max_pool_3d(inputs=relu2_2_out, filter_size=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], layer_name='pool2') # # layer3 conv3_1_out, conv3_1_weights = conv3d(inputs=pool2_out, filter_size=3, num_filters=128, num_channels=64, strides=[1, 3, 3, 3, 1], layer_name='conv3_1') relu3_1_out = relu_3d(inputs=conv3_1_out, layer_name='relu3_1') conv3_2_out, conv3_2_weights = conv3d(inputs=relu3_1_out, filter_size=3, num_filters=128, num_channels=128, strides=[1, 3, 3, 3, 1], layer_name='conv3_2') relu3_2_out = relu_3d(inputs=conv3_2_out, layer_name='relu3_2') conv3_3_out, conv3_3_weights = conv3d(inputs=relu3_2_out, filter_size=3, num_filters=128, num_channels=128, strides=[1, 3, 3, 3, 1], layer_name='conv3_3') relu3_3_out = relu_3d(inputs=conv3_3_out, layer_name='relu3_3') pool3_out = max_pool_3d(inputs=relu3_3_out, filter_size=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], layer_name='pool3') # # layer4 conv4_1_out, conv4_1_weights = conv3d(inputs=pool3_out, filter_size=3, num_filters=256, num_channels=128, strides=[1, 3, 3, 3, 1], layer_name='conv4_1') relu4_1_out = relu_3d(inputs=conv4_1_out, layer_name='relu4_1') conv4_2_out, conv4_2_weights = conv3d(inputs=relu4_1_out, filter_size=3, num_filters=256, num_channels=256, strides=[1, 3, 3, 3, 1], layer_name='conv4_2') relu4_2_out = relu_3d(inputs=conv4_2_out, layer_name='relu4_2') conv4_3_out, conv4_3_weights = conv3d(inputs=relu4_2_out, filter_size=3, num_filters=256, num_channels=256, strides=[1, 3, 3, 3, 1], layer_name='conv4_3') relu4_3_out = relu_3d(inputs=conv4_3_out, layer_name='relu4_3') pool4_out = max_pool_3d(inputs=relu4_3_out, filter_size=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], layer_name='pool4') # # layer5 conv5_1_out, conv5_1_weights = conv3d(inputs=pool4_out, filter_size=3, num_filters=512, num_channels=256, strides=[1, 3, 3, 3, 1], layer_name='conv5_1') relu5_1_out = relu_3d(inputs=conv5_1_out, layer_name='relu5_1') conv5_2_out, conv5_2_weights = conv3d(inputs=relu5_1_out, filter_size=3, num_filters=512, num_channels=512, strides=[1, 3, 3, 3, 1], layer_name='conv5_2') relu5_2_out = relu_3d(inputs=conv5_2_out, layer_name='relu5_2') conv5_3_out, conv5_3_weights = conv3d(inputs=relu5_2_out, filter_size=3, num_filters=512, num_channels=512, strides=[1, 3, 3, 3, 1], layer_name='conv5_3') relu5_3_out = relu_3d(inputs=conv5_3_out, layer_name='relu5_3') pool5_out = max_pool_3d(inputs=relu5_3_out, filter_size=[1, 2, 2, 2, 1], strides=[1, 2, 2, 2, 1], layer_name='pool5') flatten5_out, flatten5_features = flatten_3d(pool5_out, layer_name='flatten5') # layer6 dense6_out = dense_3d(inputs=flatten5_out, num_inputs=int(flatten5_out.shape[1]), num_outputs=4096, layer_name='fc6') relu6_out = relu_3d(inputs=dense6_out, layer_name='relu6') dropout6_out = dropout_3d(inputs=relu6_out, keep_prob=keep_prob, layer_name='drop6') # layer7 dense7_out = dense_3d(inputs=dropout6_out, num_inputs=int(dropout6_out.shape[1]), num_outputs=4096, layer_name='fc7') relu7_out = relu_3d(inputs=dense7_out, layer_name='relu7') dropout7_out = dropout_3d(inputs=relu7_out, keep_prob=keep_prob, layer_name='drop7') # layer8 dense8_out = dense_3d(inputs=dropout7_out, num_inputs=int(dropout7_out.shape[1]), num_outputs=1000, layer_name='fc8') # layer9 dense9_out = dense_3d(inputs=dense8_out, num_inputs=int(dense8_out.shape[1]), num_outputs=FLAGS.num_classes, layer_name='fc9') # Final softmax y = tf.nn.softmax(dense9_out) # Overall Metrics Calculations with tf.name_scope('log_loss'): log_loss = tf.losses.log_loss( y_labels, y, epsilon=10e-15 ) #+ tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) tf.summary.scalar('log_loss', log_loss) with tf.name_scope('softmax_cross_entropy'): softmax_cross_entropy = tf.losses.softmax_cross_entropy( y_labels, dense9_out ) #+ tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) tf.summary.scalar('softmax_cross_entropy', softmax_cross_entropy) with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_labels, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) tf.summary.scalar('accuracy', accuracy) with tf.name_scope('weighted_log_loss'): weighted_log_loss = tf.losses.log_loss( y_labels, y, weights=class_weights, epsilon=10e-15 ) #+ tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) tf.summary.scalar('weighted_log_loss', weighted_log_loss) with tf.name_scope('weighted_softmax_cross_entropy'): weighted_softmax_cross_entropy = tf.losses.softmax_cross_entropy( y_labels, dense9_out, weights=cross_entropy_weights) tf.summary.scalar('weighted_softmax_cross_entropy', weighted_softmax_cross_entropy) with tf.name_scope('sparse_softmax_cross_entropy'): y_labels_argmax_int = tf.to_int32(tf.argmax(y_labels, axis=1)) sparse_softmax_cross_entropy = tf.losses.sparse_softmax_cross_entropy( labels=y_labels_argmax_int, logits=dense9_out ) #+ tf.add_n(tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)) tf.summary.scalar('sparse_softmax_cross_entropy', sparse_softmax_cross_entropy) with tf.name_scope('weighted_sparse_softmax_cross_entropy'): y_labels_argmax_int = tf.to_int32(tf.argmax(y_labels, axis=1)) weighted_sparse_softmax_cross_entropy = tf.losses.sparse_softmax_cross_entropy( labels=y_labels_argmax_int, logits=dense9_out, weights=cross_entropy_weights) tf.summary.scalar('weighted_sparse_softmax_cross_entropy', weighted_sparse_softmax_cross_entropy) # Class Based Metrics calculations y_pred_class = tf.argmax(y, 1) y_labels_class = tf.argmax(y_labels, 1) confusion_matrix = tf.confusion_matrix(y_labels_class, y_pred_class, num_classes=FLAGS.num_classes) sum_row_0 = tf.reduce_sum(confusion_matrix[0, :]) sum_row_1 = tf.reduce_sum(confusion_matrix[1, :]) sum_row_2 = tf.reduce_sum(confusion_matrix[2, :]) sum_row_3 = tf.reduce_sum(confusion_matrix[3, :]) sum_row_4 = tf.reduce_sum(confusion_matrix[4, :]) #sum_row_5 = tf.reduce_sum(confusion_matrix[5, :]) sum_col_0 = tf.reduce_sum(confusion_matrix[:, 0]) sum_col_1 = tf.reduce_sum(confusion_matrix[:, 1]) sum_col_2 = tf.reduce_sum(confusion_matrix[:, 2]) sum_col_3 = tf.reduce_sum(confusion_matrix[:, 3]) sum_col_4 = tf.reduce_sum(confusion_matrix[:, 4]) #sum_col_5 = tf.reduce_sum(confusion_matrix[:, 5]) sum_all = tf.reduce_sum(confusion_matrix[:, :]) with tf.name_scope('precision'): precision_0 = confusion_matrix[0, 0] / sum_col_0 precision_1 = confusion_matrix[1, 1] / sum_col_1 precision_2 = confusion_matrix[2, 2] / sum_col_2 precision_3 = confusion_matrix[3, 3] / sum_col_3 precision_4 = confusion_matrix[4, 4] / sum_col_4 # precision_5 = confusion_matrix[5,5] / sum_col_5 tf.summary.scalar('precision_0', precision_0) tf.summary.scalar('precision_1', precision_1) tf.summary.scalar('precision_2', precision_2) tf.summary.scalar('precision_3', precision_3) tf.summary.scalar('precision_4', precision_4) # tf.summary.scalar('precision_5', precision_5) with tf.name_scope('recall'): recall_0 = confusion_matrix[0, 0] / sum_row_0 recall_1 = confusion_matrix[1, 1] / sum_row_1 recall_2 = confusion_matrix[2, 2] / sum_row_2 recall_3 = confusion_matrix[3, 3] / sum_row_3 recall_4 = confusion_matrix[4, 4] / sum_row_4 # recall_5 = confusion_matrix[5,5] / sum_row_5 tf.summary.scalar('recall_0', recall_0) tf.summary.scalar('recall_1', recall_1) tf.summary.scalar('recall_2', recall_2) tf.summary.scalar('recall_3', recall_3) tf.summary.scalar('recall_4', recall_4) # tf.summary.scalar('recall_5', recall_5) with tf.name_scope('specificity'): tn_0 = sum_all - (sum_row_0 + sum_col_0 - confusion_matrix[0, 0]) fp_0 = sum_col_0 - confusion_matrix[0, 0] specificity_0 = tn_0 / (tn_0 + fp_0) tn_1 = sum_all - (sum_row_1 + sum_col_1 - confusion_matrix[1, 1]) fp_1 = sum_col_1 - confusion_matrix[1, 1] specificity_1 = tn_1 / (tn_1 + fp_1) tn_2 = sum_all - (sum_row_2 + sum_col_2 - confusion_matrix[2, 2]) fp_2 = sum_col_2 - confusion_matrix[2, 2] specificity_2 = tn_2 / (tn_2 + fp_2) tn_3 = sum_all - (sum_row_3 + sum_col_3 - confusion_matrix[3, 3]) fp_3 = sum_col_3 - confusion_matrix[3, 3] specificity_3 = tn_3 / (tn_3 + fp_3) tn_4 = sum_all - (sum_row_4 + sum_col_4 - confusion_matrix[4, 4]) fp_4 = sum_col_4 - confusion_matrix[4, 4] specificity_4 = tn_4 / (tn_4 + fp_4) # tn_5 = sum_all - (sum_row_5 + sum_col_5 - confusion_matrix[5,5]) # fp_5 = sum_col_5 - confusion_matrix[5,5] # specificity_5 = tn_5 / (tn_5 + fp_5) tf.summary.scalar('specificity_0', specificity_0) tf.summary.scalar('specificity_1', specificity_1) tf.summary.scalar('specificity_2', specificity_2) tf.summary.scalar('specificity_3', specificity_3) tf.summary.scalar('specificity_4', specificity_4) # tf.summary.scalar('specificity_5', specificity_5) with tf.name_scope('true_positives'): tp_0 = confusion_matrix[0, 0] tp_1 = confusion_matrix[1, 1] tp_2 = confusion_matrix[2, 2] tp_3 = confusion_matrix[3, 3] tp_4 = confusion_matrix[4, 4] # tp_5 = confusion_matrix[5,5] tf.summary.scalar('true_positives_0', tp_0) tf.summary.scalar('true_positives_1', tp_1) tf.summary.scalar('true_positives_2', tp_2) tf.summary.scalar('true_positives_3', tp_3) tf.summary.scalar('true_positives_4', tp_4) # tf.summary.scalar('true_positives_5', tp_5) with tf.name_scope('true_negatives'): tf.summary.scalar('true_negatives_0', tn_0) tf.summary.scalar('true_negatives_1', tn_1) tf.summary.scalar('true_negatives_2', tn_2) tf.summary.scalar('true_negatives_3', tn_3) tf.summary.scalar('true_negatives_4', tn_4) # tf.summary.scalar('true_negatives_5', tn_5) with tf.name_scope('false_positives'): tf.summary.scalar('false_positives_0', fp_0) tf.summary.scalar('false_positives_1', fp_1) tf.summary.scalar('false_positives_2', fp_2) tf.summary.scalar('false_positives_3', fp_3) tf.summary.scalar('false_positives_4', fp_4) # tf.summary.scalar('false_positives_5', fp_5) with tf.name_scope('false_negatives'): fn_0 = sum_row_0 - tp_0 fn_1 = sum_row_1 - tp_1 fn_2 = sum_row_2 - tp_2 fn_3 = sum_row_3 - tp_3 fn_4 = sum_row_4 - tp_4 # fn_5 = sum_row_5 - tp_5 tf.summary.scalar('false_negatives_0', fn_0) tf.summary.scalar('false_negatives_1', fn_1) tf.summary.scalar('false_negatives_2', fn_2) tf.summary.scalar('false_negatives_3', fn_3) tf.summary.scalar('false_negatives_4', fn_4) # tf.summary.scalar('false_negatives_5', fn_5) with tf.name_scope('log_loss_by_class'): log_loss_0 = tf.losses.log_loss(y_labels[0], y[0], epsilon=10e-15) log_loss_1 = tf.losses.log_loss(y_labels[1], y[1], epsilon=10e-15) log_loss_2 = tf.losses.log_loss(y_labels[2], y[2], epsilon=10e-15) log_loss_3 = tf.losses.log_loss(y_labels[3], y[3], epsilon=10e-15) log_loss_4 = tf.losses.log_loss(y_labels[4], y[4], epsilon=10e-15) # log_loss_5 = tf.losses.log_loss(y_labels[5], y[5], epsilon=10e-15) #added extra '_' to avoid tenosorboard name collision with the main log_loss metric tf.summary.scalar('log_loss__0', log_loss_0) tf.summary.scalar('log_loss__1', log_loss_1) tf.summary.scalar('log_loss__2', log_loss_2) tf.summary.scalar('log_loss__3', log_loss_3) tf.summary.scalar('log_loss__4', log_loss_4) # tf.summary.scalar('log_loss__5', log_loss_5) with tf.name_scope('softmax_cross_entropy_by_class'): softmax_cross_entropy_0 = tf.losses.softmax_cross_entropy( y_labels[0], dense9_out[0]) softmax_cross_entropy_1 = tf.losses.softmax_cross_entropy( y_labels[1], dense9_out[1]) softmax_cross_entropy_2 = tf.losses.softmax_cross_entropy( y_labels[2], dense9_out[2]) softmax_cross_entropy_3 = tf.losses.softmax_cross_entropy( y_labels[3], dense9_out[3]) softmax_cross_entropy_4 = tf.losses.softmax_cross_entropy( y_labels[4], dense9_out[4]) # softmax_cross_entropy_5 = tf.losses.softmax_cross_entropy(y_labels[5], dense9_out[5]) tf.summary.scalar('softmax_cross_entropy_0', softmax_cross_entropy_0) tf.summary.scalar('softmax_cross_entropy_1', softmax_cross_entropy_1) tf.summary.scalar('softmax_cross_entropy_2', softmax_cross_entropy_2) tf.summary.scalar('softmax_cross_entropy_3', softmax_cross_entropy_3) tf.summary.scalar('softmax_cross_entropy_4', softmax_cross_entropy_4) # tf.summary.scalar('softmax_cross_entropy_5', softmax_cross_entropy_5) with tf.name_scope('accuracy_by_class'): accuracy_0 = (tp_0 + tn_0) / (tp_0 + fp_0 + fn_0 + tn_0) accuracy_1 = (tp_1 + tn_1) / (tp_1 + fp_1 + fn_1 + tn_1) accuracy_2 = (tp_2 + tn_2) / (tp_2 + fp_2 + fn_2 + tn_2) accuracy_3 = (tp_3 + tn_3) / (tp_3 + fp_3 + fn_3 + tn_3) accuracy_4 = (tp_4 + tn_4) / (tp_4 + fp_4 + fn_4 + tn_4) # accuracy_5 = (tp_5 + tn_5)/(tp_5 + fp_5 + fn_5 + tn_5) tf.summary.scalar('accuracy_0', accuracy_0) tf.summary.scalar('accuracy_1', accuracy_1) tf.summary.scalar('accuracy_2', accuracy_2) tf.summary.scalar('accuracy_3', accuracy_3) tf.summary.scalar('accuracy_4', accuracy_4) # tf.summary.scalar('accuracy_5', accuracy_5) with tf.name_scope('weighted_log_loss_by_class'): weighted_log_loss_0 = tf.losses.log_loss(y_labels[0], y[0], weights=class_weights[0], epsilon=10e-15) weighted_log_loss_1 = tf.losses.log_loss(y_labels[1], y[1], weights=class_weights[1], epsilon=10e-15) weighted_log_loss_2 = tf.losses.log_loss(y_labels[2], y[2], weights=class_weights[2], epsilon=10e-15) weighted_log_loss_3 = tf.losses.log_loss(y_labels[3], y[3], weights=class_weights[3], epsilon=10e-15) weighted_log_loss_4 = tf.losses.log_loss(y_labels[4], y[4], weights=class_weights[4], epsilon=10e-15) # weighted_log_loss_5 = tf.losses.log_loss(y_labels[5], y[5], weights=class_weights[5], epsilon=10e-15) tf.summary.scalar('weighted_log_loss_0', weighted_log_loss_0) tf.summary.scalar('weighted_log_loss_1', weighted_log_loss_1) tf.summary.scalar('weighted_log_loss_2', weighted_log_loss_2) tf.summary.scalar('weighted_log_loss_3', weighted_log_loss_3) tf.summary.scalar('weighted_log_loss_4', weighted_log_loss_4) # tf.summary.scalar('weighted_log_loss_5', weighted_log_loss_5) with tf.name_scope('f1_score_by_class'): f1_score_0 = 2 * (precision_0 * recall_0) / (precision_0 + recall_0) f1_score_1 = 2 * (precision_1 * recall_1) / (precision_1 + recall_1) f1_score_2 = 2 * (precision_2 * recall_2) / (precision_2 + recall_2) f1_score_3 = 2 * (precision_3 * recall_3) / (precision_3 + recall_3) f1_score_4 = 2 * (precision_4 * recall_4) / (precision_4 + recall_4) # f1_score_5 = 2 * (precision_5 * recall_5) / (precision_5 + recall_5) #f1_score = (f1_score_0 * 40591.0/69920.0) + (f1_score_1 * 14624.0/69920.0) + (f1_score_2 * 10490.0/69920.0) + (f1_score_3 *4215.0/ 69920.0) tf.summary.scalar('f1_score_0', f1_score_0) tf.summary.scalar('f1_score_1', f1_score_1) tf.summary.scalar('f1_score_2', f1_score_2) tf.summary.scalar('f1_score_3', f1_score_3) tf.summary.scalar('f1_score_4', f1_score_4) # tf.summary.scalar('f1_score_5', f1_score_5) with tf.name_scope('train'): optimizer = tf.train.AdamOptimizer( learning_rate=1e-4, name='adam_optimizer').minimize(log_loss) merged = tf.summary.merge_all() saver = tf.train.Saver() # Setting up config config = tf.ConfigProto() config.gpu_options.allow_growth = FLAGS.allow_growth config.log_device_placement = FLAGS.log_device_placement config.allow_soft_placement = FLAGS.allow_soft_placement # timestamp used to identify the start of run start_timestamp = str(int(time.time())) model_id = str(uuid.uuid4()) # Name used to save all artifacts of run run_name = 'runType={0:}_timestamp={1:}_batchSize={2:}_maxIterations={3:}_numTrain={4:}_numValidation={5:}_modelId={6:}' train_run_name = run_name.format('train', start_timestamp, FLAGS.batch_size, FLAGS.max_iterations, train_x.shape[0], validation_x.shape[0], model_id) test_run_name = run_name.format('test', start_timestamp, FLAGS.batch_size, FLAGS.max_iterations, train_x.shape[0], validation_x.shape[0], model_id) print('Run_name: {}'.format(train_run_name)) k_count = 0 with tf.Session(graph=graph, config=config) as sess: train_writer = tf.summary.FileWriter( TENSORBOARD_SUMMARIES + train_run_name, sess.graph) test_writer = tf.summary.FileWriter( TENSORBOARD_SUMMARIES + test_run_name, sess.graph) sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) for i in tqdm(range(FLAGS.max_iterations)): if (i % FLAGS.iteration_analysis == 0) or (i == (FLAGS.max_iterations - 1)): save_model(sess, model_id, saver) # Validation num_batches = int( math.ceil(float(len(validation_x)) / FLAGS.batch_size)) for k in range(num_batches): _, step_summary = sess.run([y, merged], feed_dict=feed_dict(False, k)) test_writer.add_summary(step_summary, k_count) k_count = k_count + 1 else: # Train _, step_summary = sess.run([optimizer, merged], feed_dict=feed_dict(True)) train_writer.add_summary(step_summary, i) train_writer.close() test_writer.close() # Clossing session sess.close()
def model_fn(mode, inputs, params, reuse=False): """Model function defining the graph operations. Args: mode: (string) 'train', 'eval', etc. inputs: (dict) contains the inputs of the graph (features, labels...) this can be `tf.placeholder` or outputs of `tf.data` params: (Params) contains hyperparameters of the model (ex: `params.learning_rate`) reuse: (bool) whether to reuse the weights Returns: model_spec: (dict) contains the graph operations or nodes needed for training / evaluation """ is_training = (mode == 'train') labels = inputs['labels'] sentence_lengths = inputs['sentence_lengths'] # ----------------------------------------------------------- # MODEL: define the layers of the model with tf.variable_scope('model', reuse=reuse): # Compute the output distribution of the model and the predictions logits = build_model(mode, inputs, params, is_training) predictions = tf.cast(tf.argmax(logits, -1), tf.int32) #labels = tf.reshape(labels, [-1,1]) #predictions = [1 for i in logits > 0 else 0] # Define loss and accuracy (we need to apply a mask to account for padding) losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) print(losses) print(labels) losses = tf.multiply(tf.to_float(tf.add(3 * labels, 1)), losses) loss = tf.reduce_mean(losses) accuracy = tf.reduce_mean( tf.cast(tf.equal(labels, predictions), tf.float32)) # Define training step that minimizes the loss with the Adam optimizer if is_training: optimizer = tf.train.AdamOptimizer(params.learning_rate) global_step = tf.train.get_or_create_global_step() train_op = optimizer.minimize(loss, global_step=global_step) # ----------------------------------------------------------- # METRICS AND SUMMARIES # Metrics for evaluation using tf.metrics (average over whole dataset) with tf.variable_scope("metrics"): metrics = { 'accuracy': tf.metrics.accuracy(labels=labels, predictions=predictions), 'loss': tf.metrics.mean(loss), 'recall': tf.metrics.recall(labels=labels, predictions=predictions), 'precision': tf.metrics.precision(labels=labels, predictions=predictions) } # Group the update ops for the tf.metrics update_metrics_op = tf.group(*[op for _, op in metrics.values()]) # Get the op to reset the local variables used in tf.metrics metric_variables = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="metrics") metrics_init_op = tf.variables_initializer(metric_variables) # Summaries for training tf.summary.scalar('loss', loss) tf.summary.scalar('accuracy', accuracy) # ----------------------------------------------------------- # MODEL SPECIFICATION # Create the model specification and return it # It contains nodes or operations in the graph that will be used for training and evaluation model_spec = inputs variable_init_op = tf.group( *[tf.global_variables_initializer(), tf.tables_initializer()]) model_spec['variable_init_op'] = variable_init_op model_spec["predictions"] = predictions model_spec['loss'] = loss model_spec['accuracy'] = accuracy model_spec['metrics_init_op'] = metrics_init_op model_spec['metrics'] = metrics model_spec['update_metrics'] = update_metrics_op model_spec['summary_op'] = tf.summary.merge_all() model_spec['matrix'] = tf.confusion_matrix(labels=labels, predictions=predictions) if is_training: model_spec['train_op'] = train_op return model_spec
logits=dense_layer3, labels=y) # apply the weights, relying on broadcasting of the multiplication weighted_losses = unweighted_losses * weights # reduce the result to get your final loss loss = tf.reduce_mean(weighted_losses) #cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=dense_layer3, labels=y)) optimiser = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss) # define an accuracy assessment operation correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) pred = tf.cast(tf.argmax(y_, 1), tf.float32) label = tf.cast(tf.argmax(y, 1), tf.float32) auc = tf.metrics.auc(pred, label, curve='ROC') confusion = tf.confusion_matrix(labels=label, predictions=pred, num_classes=2) ########################################################################################### for set_i in range(start_set, end_set): test_receptor = receptor_list[set_i] print('test receptor: ', test_receptor) training_set_0 = [] training_set_1 = [] test_set_0 = [] test_set_1 = [] for k in ligand_dict.keys(): if k.split('_')[0] in image_dict.keys(): if k.split('_')[0] != test_receptor: if k.endswith('_0'):
feed_dict={ hycor.input_x: test_data, hycor.input_y: test_label, hycor.seqlen: test_seqs, hycor.dropout_keep_prob: 1.0 })) # get actual labels actual = np.array([np.where(r == 1)[0][0] for r in test_label]) predicted = hycor.logits.eval(feed_dict={ hycor.input_x: test_data, hycor.dropout_keep_prob: 1.0 }) print('Confusion Matrix: (H:labels, V:Predictions)') cm = tf.confusion_matrix(actual, predicted, num_classes=y_train.shape[1]) # get confusion matrix values var_cm = sess.run(cm) print(var_cm) accuracy = np.sum([var_cm[i, i] for i in range(var_cm.shape[1]) ]) / np.sum(var_cm) # normalize confusion matrixA print('Precision | Recall | Fscore') if (y_train.shape[1] == 2): print(calcMetric.pre_rec_fs2(var_cm)) elif (y_train.shape[1] == 3): print(calcMetric.pre_rec_fs3(var_cm)) elif (y_train.shape[1] == 4): print(calcMetric.pre_rec_fs4(var_cm)) elif (y_train.shape[1] == 5):
def run_inference(wanted_words, sample_rate, clip_duration_ms, window_size_ms, window_stride_ms, dct_coefficient_count, model_architecture, model_size_info): """Creates an audio model with the nodes needed for inference. Uses the supplied arguments to create a model, and inserts the input and output nodes that are needed to use the graph for inference. Args: wanted_words: Comma-separated list of the words we're trying to recognize. sample_rate: How many samples per second are in the input audio files. clip_duration_ms: How many samples to analyze for the audio pattern. window_size_ms: Time slice duration to estimate frequencies from. window_stride_ms: How far apart time slices should be. dct_coefficient_count: Number of frequency bands to analyze. model_architecture: Name of the kind of model to generate. model_size_info: Model dimensions : different lengths for different models """ tf.logging.set_verbosity(tf.logging.INFO) sess = tf.InteractiveSession() words_list = input_data.prepare_words_list(wanted_words.split(',')) model_settings = models.prepare_model_settings( len(words_list), sample_rate, clip_duration_ms, window_size_ms, window_stride_ms, dct_coefficient_count) audio_processor = input_data.AudioProcessor( FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage, FLAGS.wanted_words.split(','), FLAGS.validation_percentage, FLAGS.testing_percentage, model_settings) label_count = model_settings['label_count'] fingerprint_size = model_settings['fingerprint_size'] fingerprint_input = tf.placeholder( tf.float32, [None, fingerprint_size], name='fingerprint_input') logits = models.create_model( fingerprint_input, model_settings, FLAGS.model_architecture, FLAGS.model_size_info, is_training=False) ground_truth_input = tf.placeholder( tf.float32, [None, label_count], name='groundtruth_input') predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix( expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) models.load_variables_from_checkpoint(sess, FLAGS.checkpoint) # training set set_size = audio_processor.set_size('training') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): training_fingerprints, training_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'training', sess)) training_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: training_fingerprints, ground_truth_input: training_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (training_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Training accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size)) # validation set set_size = audio_processor.set_size('validation') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): validation_fingerprints, validation_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'validation', sess)) validation_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: validation_fingerprints, ground_truth_input: validation_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (validation_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Validation accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size)) # test set set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): test_fingerprints, test_ground_truth = audio_processor.get_data( FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess) test_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (test_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Test accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size))
def model(X_train, Y_train, X_test, Y_test, Y_train_labels, Y_test_labels, seg_size, weights, decay_learning_rate=False, base_learning_rate=0.0001, num_epochs=100, minibatch_size=16, print_cost=True): """ Implements a three-layer tensorflow neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SOFTMAX. Arguments: X_train -- training set Y_train -- test set X_test -- training set Y_test -- test set learning_rate -- learning rate of the optimization num_epochs -- number of epochs of the optimization loop minibatch_size -- size of a minibatch print_cost -- True to print the cost every 100 epochs Returns: parameters -- parameters learnt by the model. They can then be used to predict. """ ops.reset_default_graph( ) # to be able to rerun the model without overwriting tf variables (m, seg_size, n_C0) = (X_train.channels.shape) # calculating the learning rate type batch = tf.Variable(0, dtype=tf.float32) if decay_learning_rate is True: # Decay once per epoch, using an exponential schedule starting at 0.01. learning_rate = tf.train.exponential_decay( 0.01, # Base learning rate. batch * minibatch_size, # Current index into the dataset. m, # Decay step. 0.95, # Decay rate. staircase=True) else: # Decay once per epoch, using an exponential schedule starting at 0.01. learning_rate = tf.train.exponential_decay( base_learning_rate, # Base learning rate. batch * minibatch_size, # Current index into the dataset. m, # Decay step. 1.00, # Decay rate. staircase=True) n_y = num_classes costs = [] # variable for saving trip_id, uuid, segment_id fro test set predicted_label = np.zeros(Y_test.shape[0], dtype=[('uuid', 'S64'), ('trip_id', 'int8'), ('segment_id', 'int8'), ('class_label', 'int8')]) predicted_label = np.rec.array(predicted_label) predicted_label.uuid = Y_test.uuid predicted_label.trip_id = Y_test.trip_id predicted_label.segment_id = Y_test.segment_id # variable for saving trip_id, uuid, segment_id for train set predicted_label_train = np.zeros(Y_train.shape[0], dtype=[('uuid', 'S64'), ('trip_id', 'int8'), ('segment_id', 'int8'), ('class_label', 'int8')]) predicted_label_train = np.rec.array(predicted_label_train) predicted_label_train.uuid = Y_train.uuid predicted_label_train.trip_id = Y_train.trip_id predicted_label_train.segment_id = Y_train.segment_id # Create Placeholders of the correct shape X, Y, minibatch_weights = create_placeholders( seg_size, n_C0, n_y, minibatch_size=minibatch_size) # Initialize parameters parameters = initialize_parameters(weights) # Forward propagation: Build the forward propagation in the tensorflow graph final_Z = forward_propagation(X, parameters) # Cost function: Add cost function to tensorflow graph cost = compute_cost(final_Z, Y, minibatch_weights) # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer. optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize( cost, global_step=batch) # Initialize all the variables init = tf.global_variables_initializer() # Start the session to compute the tensorflow graph with tf.Session() as sess: # Run the initialization sess.run(init) # Do the training loop for epoch in range(num_epochs): minibatch_cost = 0. num_minibatches = int( m / minibatch_size ) # number of minibatches of size minibatch_size in the train set minibatches, mini_batches_weights = random_mini_batches( X_train, Y_train, Y_train_labels, Y_test_labels, minibatch_size, class_weight_calculation=0) for index, minibatch in enumerate(minibatches): # Select a minibatch (minibatch_X, minibatch_Y) = minibatch class_weights = mini_batches_weights[index] # IMPORTANT: The line that runs the graph on a minibatch. # Run the session to execute the optimizer and the cost, the feedict should contain a minibatch for (X,Y). _, temp_cost = sess.run( [optimizer, cost], feed_dict={ X: minibatch_X.channels, Y: minibatch_Y.class_label, minibatch_weights: class_weights }) # _, temp_cost = sess.run(tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(compute_cost), # feed_dict={X: minibatch_X, Y: minibatch_Y, class_weights: class_weights}) minibatch_cost += temp_cost / num_minibatches # Print the cost every epoch if print_cost == True and epoch % 5 == 0: print("Cost after epoch %i: %f" % (epoch, minibatch_cost)) if print_cost == True and epoch % 1 == 0: costs.append(minibatch_cost) if minibatch_cost > costs[epoch - 1]: learning_rate = learning_rate * 0.95 # Calculate the correct predictions predict_op = tf.argmax(final_Z, 1) correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1)) # finding the probabilities final_prob_train = (final_Z.eval({ X: X_train.channels, Y: Y_train.class_label })) final_prob_test = sess.run(final_Z, feed_dict={X: X_test.channels}) # finding the predicted labels predictions, labels_test = sess.run( [predict_op, tf.argmax(Y, 1)], feed_dict={ X: X_test.channels, Y: Y_test.class_label }) # predicted_label.class_label = predictions predictions_train = (predict_op.eval({ X: X_train.channels, Y: Y_train.class_label })) # Calculate accuracy on the test set accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float")) train_accuracy = accuracy.eval({ X: X_train.channels, Y: Y_train.class_label }) test_accuracy = accuracy.eval({ X: X_test.channels, Y: Y_test.class_label }) print("Train Accuracy:", train_accuracy) print("Test Accuracy:", test_accuracy) confusion = tf.confusion_matrix(labels=tf.argmax(Y, 1), predictions=predict_op, num_classes=num_classes) confusion_mat = confusion.eval({ Y: Y_test.class_label, X: X_test.channels }) print(confusion_mat) return parameters, test_accuracy, costs, predictions, predictions_train, final_prob_train, final_prob_test
logits = models.create_model(fingerprint_input, model_settings, model_architecture, is_training=False) softmax = tf.nn.softmax(logits, name='labels_softmax') with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=ground_truth_input, logits=logits)) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) global_step = tf.train.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) tf.global_variables_initializer().run() if start_checkpoint: models.load_variables_from_checkpoint(sess, start_checkpoint) start_step = global_step.eval(session=sess) #********************************************************************* print(" ***************** audio processor ********************") training_datas = len(audio_processor.data_index['training']) + len( audio_processor.unknown_index['training']) validation_datas = len(audio_processor.data_index['validation']) + len(
def train(sess, data, n_epochs, batch_size, summaries_op, accuracy_summary_op, train_writer, test_writer, X, Y, SIFT, train_op, loss_op, accuracy_op, ratio=[70, 20, 10], N=N_CLUSTER): # record starting time train_start = time.time() saver = tf.train.Saver() # train_data, test_data, validation_data = split_data(data, ratio) X_train, X_test, y_train, y_test, SIFT_train, SIFT_test = train_test_split( data[0], data[1], data[2], test_size=0.30) train_data = [X_train, y_train] test_data = [X_test, y_test] # if N > SIFT_train.shape[0]: # N = int(sqrt(SIFT_train.shape[0])) # Step 1 : Extract key points and descriptors data_kp_train = extractKeyPoints(SIFT_train) data_kp_test = extractKeyPoints(SIFT_test) # Get descriptors and get clusters clusterer = getCluster(data_kp_train, n_cluster=N) # Get bag of features count for each image SIFT_data_train = convert2bagOfFeatures(data_kp_train, clusterer, n_cluster=N) SIFT_data_test = convert2bagOfFeatures(data_kp_test, clusterer, n_cluster=N) # Append result to train_data train_data.append(SIFT_data_train / N) test_data.append(SIFT_data_test / N) # Run through the entire dataset n_training_epochs times train_loss = 100 for i in range(n_epochs): # Initialise statistics training_loss = 0 epoch_start = time.time() batches = get_batch(train_data, batch_size) t_batches = get_batch(test_data, 10) n_batches = len(batches) # # Run the SGD train op for each minibatch for j in range(n_batches): batch = batches[j] # Run a training step trainstep_result, batch_loss, summary = \ sess.run([train_op, loss_op, summaries_op], feed_dict={X: batch[0], Y: batch[1], SIFT: batch[2]}) train_writer.add_summary(summary, j) training_loss += batch_loss # Timing and statistics epoch_duration = round(time.time() - epoch_start, 2) ave_train_loss = training_loss / n_batches # Get accuracy train_accuracy = \ accuracy(sess, train_data, batches, batch_size, X, Y, accuracy_op) test_accuracy = \ accuracy(sess, test_data, t_batches, batch_size, X, Y, accuracy_op) if train_loss > ave_train_loss: save_path = saver.save(sess, "./models/model.ckpt") train_loss = ave_train_loss print("saved checkpoint") # log accuracy at the current epoch on training and test sets train_acc_summary = sess.run( accuracy_summary_op, feed_dict={accuracy_placeholder: train_accuracy}) train_writer.add_summary(train_acc_summary, i) test_acc_summary = sess.run( accuracy_summary_op, feed_dict={accuracy_placeholder: test_accuracy}) test_writer.add_summary(test_acc_summary, i) [writer.flush() for writer in [train_writer, test_writer]] train_duration = round(time.time() - train_start, 2) # Output to montior training print('Epoch {0}, Training Loss: {1:.5f}, Test accuracy: {2:.5f}, \ time: {3}s, total time: {4}s'.format(i, ave_train_loss, test_accuracy, epoch_duration, train_duration)) print('Total training time: {0}s'.format(train_duration)) print('Confusion Matrix:') true_class = tf.argmax(Y, 1) predicted_class = tf.argmax(preds_op, 1) cm = tf.confusion_matrix(predicted_class, true_class) print( sess.run(cm, feed_dict={ X: test_data[0], Y: test_data[1], SIFT: test_data[2] }))
y = base_graph.get_tensor_by_name('y_placeholder:0') keep_prob = base_graph.get_tensor_by_name('keep_prob_placeholder:0') base_weights = base_graph.get_tensor_by_name('op7:0') base_feed_dict = {x: test_real_X[0:1], keep_prob: 1.0} base_array = ((sess1.run(base_weights, base_feed_dict))[0]) print base_array #print 'Prediction',sess1.run(tf.argmax(prediction,1), feed_dict={x:testtest, keep_prob:1}) base_change = np.argmax(base_array) print base_change print np.argmax(test_real_y[0]) #true_y = np prediction, truth = [], [] for i in range(50): base_feed_dict = {x: test_real_X[i:i + 1], keep_prob: 1.0} base_array = ((sess1.run(base_weights, base_feed_dict))[0]) base_change = np.argmax(base_array) real_base = np.argmax(test_real_y[0]) prediction.append(base_change) truth.append(real_base) print "Test %i complete" % (i + 1) confusion = tf.confusion_matrix(labels=truth, predictions=prediction, num_classes=4) sess = tf.Session() with sess.as_default(): print confusion.eval()
# Evaluate model with tf.name_scope('accuracy_calculation'): correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) tf.summary.scalar('scalar_accuracy', accuracy) # Saver Class saver = tf.train.Saver(max_to_keep=1) # Confusion matrix with tf.name_scope('confussion_matrix'): confusion = tf.confusion_matrix( labels=tf.argmax(y, 1), predictions=tf.argmax(pred, 1), num_classes=2, ) # Initializing the variables init = tf.global_variables_initializer() # Saver Class # how many checkpoints (last models) it keeps saver = tf.train.Saver(max_to_keep=1) # Launch the graph with tf.Session() as sess: merged = tf.summary.merge_all() train_writer = tf.summary.FileWriter( os.path.join( os.getcwd(), KEY_PARAMETERS["log_dir"],
def get_confusion_matrix(labels, predictions): ''' Generate a confusion matrix for evaluation. Note for rows, columns 0...5 0=NONSTICK, 1=12-STICKY...up to 5=STICK_PALINDROME :param labels: A tensor containing actual labels for each example :param predictions: A tensor containing the label predicted by the model :return: Tuple containing matrix and update op for use in eval_metric_ops ''' with tf.variable_scope('get_confusion_matrix'): matrix = tf.confusion_matrix(labels=labels, predictions=predictions, num_classes=6) matrix_sum = tf.Variable(tf.zeros(shape=(6, 6), dtype=tf.int32), trainable=False, name='confusion_matrix', collections=[tf.GraphKeys.LOCAL_VARIABLES]) # Update matrix_sum by adding matrix to it update = tf.assign_add(matrix_sum, matrix) # Return confusion matrix and update op return tf.convert_to_tensor(matrix_sum), update def cnn_model_fn(features, labels, mode): """Model function for CNN.""" # Input Layer # Reshape X to 4-D tensor: [batch_size, width, height, channels] # MNIST images are 28x28 pixels, and have one color channel input_layer = tf.reshape(features["x"], [-1, 28, 28, 1]) # Convolutional Layer #1 # Computes 32 features using a 5x5 filter with ReLU activation. # Padding is added to preserve width and height. # Input Tensor Shape: [batch_size, 28, 28, 1] # Output Tensor Shape: [batch_size, 28, 28, 32] conv1 = tf.layers.conv2d(inputs=input_layer, filters=32, kernel_size=[5, 5], padding="same", activation=tf.nn.relu) # Pooling Layer #1 # First max pooling layer with a 2x2 filter and stride of 2 # Input Tensor Shape: [batch_size, 28, 28, 32] # Output Tensor Shape: [batch_size, 14, 14, 32] pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2) # Convolutional Layer #2 # Computes 64 features using a 5x5 filter. # Padding is added to preserve width and height. # Input Tensor Shape: [batch_size, 14, 14, 32] # Output Tensor Shape: [batch_size, 14, 14, 64] conv2 = tf.layers.conv2d(inputs=pool1, filters=64, kernel_size=[5, 5], padding="same", activation=tf.nn.relu) # Pooling Layer #2 # Second max pooling layer with a 2x2 filter and stride of 2 # Input Tensor Shape: [batch_size, 14, 14, 64] # Output Tensor Shape: [batch_size, 7, 7, 64] pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2) # Flatten tensor into a batch of vectors # Input Tensor Shape: [batch_size, 7, 7, 64] # Output Tensor Shape: [batch_size, 7 * 7 * 64] pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64]) # Dense Layer # Densely connected layer with 1024 neurons # Input Tensor Shape: [batch_size, 7 * 7 * 64] # Output Tensor Shape: [batch_size, 1024] dense = tf.layers.dense(inputs=pool2_flat, units=1024, activation=tf.nn.relu) # Add dropout operation; 0.6 probability that element will be kept dropout = tf.layers.dropout( inputs=dense, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) # Logits layer # Input Tensor Shape: [batch_size, 1024] # Output Tensor Shape: [batch_size, 10] logits = tf.layers.dense(inputs=dropout, units=10) predictions = { # Generate predictions (for PREDICT and EVAL mode) "classes": tf.argmax(input=logits, axis=1), # Add `softmax_tensor` to the graph. It is used for PREDICT and by the # `logging_hook`. "probabilities": tf.nn.softmax(logits, name="softmax_tensor") } if mode == tf.estimator.ModeKeys.PREDICT: return tf.estimator.EstimatorSpec(mode=mode, predictions=predictions) # Calculate Loss (for both TRAIN and EVAL modes) onehot_labels = tf.one_hot(indices=tf.cast(labels, tf.int32), depth=10) loss = tf.losses.softmax_cross_entropy(onehot_labels=onehot_labels, logits=logits) # Configure the Training Op (for TRAIN mode) if mode == tf.estimator.ModeKeys.TRAIN: optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001) train_op = optimizer.minimize( loss=loss, global_step=tf.train.get_global_step()) return tf.estimator.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) # Add evaluation metrics (for EVAL mode) eval_metric_ops = { "accuracy": tf.metrics.accuracy(labels=labels, predictions=predictions["classes"]) } return tf.estimator.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops)
def run_quant_inference(wanted_words, sample_rate, clip_duration_ms, window_size_ms, window_stride_ms, dct_coefficient_count, model_architecture, model_size_info): """Creates an audio model with the nodes needed for inference. Uses the supplied arguments to create a model, and inserts the input and output nodes that are needed to use the graph for inference. Args: wanted_words: Comma-separated list of the words we're trying to recognize. sample_rate: How many samples per second are in the input audio files. clip_duration_ms: How many samples to analyze for the audio pattern. window_size_ms: Time slice duration to estimate frequencies from. window_stride_ms: How far apart time slices should be. dct_coefficient_count: Number of frequency bands to analyze. model_architecture: Name of the kind of model to generate. model_size_info: Model dimensions : different lengths for different models """ tf.logging.set_verbosity(tf.logging.INFO) sess = tf.InteractiveSession() words_list = input_data.prepare_words_list(wanted_words.split(',')) model_settings = models.prepare_model_settings( len(words_list), sample_rate, clip_duration_ms, window_size_ms, window_stride_ms, dct_coefficient_count) audio_processor = input_data.AudioProcessor(FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage, FLAGS.wanted_words.split(','), FLAGS.validation_percentage, FLAGS.testing_percentage, model_settings) label_count = model_settings['label_count'] fingerprint_size = model_settings['fingerprint_size'] fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size], name='fingerprint_input') logits = models.create_model(fingerprint_input, model_settings, FLAGS.model_architecture, FLAGS.model_size_info, FLAGS.act_max, is_training=False) # Save graph.pbtxt. tf.train.write_graph(sess.graph_def, "./", FLAGS.model_architecture + '_quantized.pbtxt') ground_truth_input = tf.placeholder(tf.float32, [None, label_count], name='groundtruth_input') predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) models.load_variables_from_checkpoint(sess, FLAGS.checkpoint) # Quantize weights to 8-bits using (min,max) and write to file f = open('weights.h', 'wb') f.close() for v in tf.trainable_variables(): var_name = str(v.name) var_values = sess.run(v) min_value = var_values.min() max_value = var_values.max() int_bits = int(np.ceil(np.log2(max(abs(min_value), abs(max_value))))) dec_bits = 7 - int_bits # convert to [-128,128) or int8 var_values = np.round(var_values * 2**dec_bits) var_name = var_name.replace('/', '_') var_name = var_name.replace(':', '_') with open('weights.h', 'a') as f: f.write('#define ' + var_name + ' {') with open('weights.h', 'ab') as f: np.savetxt(f, var_values.transpose(), fmt='%d', delimiter=', ', newline=', ') with open('weights.h', 'a') as f: f.write('}\n') # convert back original range but quantized to 8-bits or 256 levels var_values = var_values / (2**dec_bits) # update the weights in tensorflow graph for quantizing the activations var_values = sess.run(tf.assign(v, var_values)) print(var_name+' number of wts/bias: '+str(var_values.shape)+\ ' dec bits: '+str(dec_bits)+\ ' max: ('+str(var_values.max())+','+str(max_value)+')'+\ ' min: ('+str(var_values.min())+','+str(min_value)+')') # training set set_size = audio_processor.set_size('training') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): training_fingerprints, training_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'training', sess)) training_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: training_fingerprints, ground_truth_input: training_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (training_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Training accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size)) # validation set set_size = audio_processor.set_size('validation') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): validation_fingerprints, validation_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'validation', sess)) validation_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: validation_fingerprints, ground_truth_input: validation_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (validation_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Validation accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size)) # test set set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): test_fingerprints, test_ground_truth = audio_processor.get_data( FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess) test_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (test_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Test accuracy = %.2f%% (N=%d)' % (total_accuracy * 100, set_size))
train_writer.add_summary(summary, i) print('test accuracy %g' % accuracy.eval( feed_dict={ x: dataset.test_images, y_: dataset.test_labels, keep_prob_1: 1.0, keep_prob_2: 1.0, keep_prob_3: 1.0, keep_prob_4: 1.0 })) predictions = sess.run(y_conv, feed_dict={ x: dataset.test_images, keep_prob_1: 1.0, keep_prob_2: 1.0, keep_prob_3: 1.0, keep_prob_4: 1.0 }) # print (predictions) true_class = np.argmax(dataset.test_labels, 1) predicted_class = np.argmax(predictions, 1) cm = tf.confusion_matrix(predicted_class, true_class) print(cm) import ipdb ipdb.set_trace() train_writer.close()
def _train(self): with tf.variable_scope('train'): """ lables = [batch_size, max_sentence_count] logits = [batch_size, max_sentence_count, 4] corss_entropy = [batch_size, max_sentence_count] """ self.cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( name='cross_entropy', labels=self.labels, logits=self.logits) """ compute regularizer """ tvars = tf.trainable_variables() self.regularization_loss = 0 for tvar in tvars: self.regularization_loss += tf.nn.l2_loss( name='l2_regularizer', t=tvar) """ Here we are trying to calculate the average loss for the entire batch. Before doing this we try to reduce the class imbalance by multiplying class_weights with cross_entropy. Not sure how much helpful this is for reducing class imbalance. """ # self.loss = tf.reduce_mean(name = 'loss', # input_tensor = tf.multiply(self.cross_entropy, self.label_weights, # name = 'mul_label_weights')) self.loss = tf.reduce_mean( name='loss', input_tensor=tf.multiply(self.cross_entropy, self.label_weights, name='mul_label_weights') ) + self.args.reg_constant * self.regularization_loss # self.loss = tf.reduce_mean(name = 'loss', input_tensor = self.cross_entropy) tf.summary.scalar('train_loss', self.loss) """ compute accuracy """ # TODO: make sure to exclude padded values while calculating accuracy by using some mask _, self.accuracy = tf.metrics.accuracy( name='accuracy', labels=self.labels, predictions=self.predictions, weights=self.sentence_mask) tf.summary.scalar('train_accuracy', self.accuracy) """ compute confusion matrix and f1 scores """ flat_labels = tf.reshape( name='flat_labels', tensor=self.labels, shape=[self.batch_size * self.max_sentence_count]) flat_predictions = tf.reshape( name='flat_predictions', tensor=self.predictions, shape=[self.batch_size * self.max_sentence_count]) flat_label_weights = tf.reshape( name='flat_label_weights', tensor=self.sentence_mask, shape=[self.batch_size * self.max_sentence_count]) self.confusion_matrix = tf.confusion_matrix( name='confusion_matrix', labels=flat_labels, predictions=flat_predictions, weights=flat_label_weights, num_classes=self.args.num_classes) self.f1_score_0 = compute_f1_score(self.confusion_matrix, class_id=0, scope='f1_score_0') tf.summary.scalar('f1_score_0', self.f1_score_0) self.f1_score_1 = compute_f1_score(self.confusion_matrix, class_id=1, scope='f1_score_1') tf.summary.scalar('f1_score_1', self.f1_score_1) self.f1_score_2 = compute_f1_score(self.confusion_matrix, class_id=2, scope='f1_score_2') tf.summary.scalar('f1_score_2', self.f1_score_2) self.f1_score_3 = compute_f1_score(self.confusion_matrix, class_id=3, scope='f1_score_3') tf.summary.scalar('f1_score_3', self.f1_score_3) self.f1_score = (self.f1_score_0 + self.f1_score_1 + self.f1_score_2 + self.f1_score_3) / 4 tf.summary.scalar('f1_score', self.f1_score) # """ # compute f1 score # """ # _, self.precision = tf.metrics.precision(name = 'precision', labels = self.labels, # predictions = self.predictions, # weights = self.label_weights) # _, self.recall = tf.metrics.recall(name = 'recall', labels = self.labels, # predictions = self.predictions, # weights = self.label_weights) # self.f1_score = 2 * self.precision * self.recall / (self.precision + self.recall) # tf.summary.scalar('train_f1_score', self.f1_score) """ Get all the trainable variables. Define gradient of loss wrt to these trainable variables. Clip gradients. """ grads, global_norm = tf.clip_by_global_norm( tf.gradients(self.loss, tvars), self.args.max_grad_norm) tf.summary.scalar('global_grad_norm', global_norm) """ Apply gradients """ opt = tf.train.AdamOptimizer(self.args.lr) self.train_op = opt.apply_gradients(name='train_op', grads_and_vars=zip( grads, tvars), global_step=self.global_step) """ Merge all summaries """ self.summary_op = tf.summary.merge_all()
y_est = tf.add(tf.matmul(x_fc1_drop, W_fc2), b_fc2) # Probabilities - output from model (not the same as logits) y_ = tf.nn.softmax(y_est) learning_rate = 1e-6 cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2(labels=y, logits=y_est)) Optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(cross_entropy) # Setup to test accuracy of model correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) con_mat = tf.confusion_matrix(labels=tf.argmax(y, 1), predictions=tf.argmax(y_, 1), num_classes=4, dtype=tf.int32) sess.run(tf.global_variables_initializer()) def get_one_hot(lb): labels_indexes = { 'stage i': 0, 'stage ii': 1, 'stage iii': 2, 'stage iv': 3, } label = np.zeros(4) label[labels_indexes[lb]] = 1 return label
def main(): # 读取所有图片 image_lists = create_image_lists(TEST_PERCENTAGE, VALIDATION_PERCENTAGE) n_classes = len(image_lists.keys()) # 5种花,该值为5 # 载入已有的模型 with gfile.FastGFile(os.path.join(MODEL_DIR, MODEL_FILE), 'rb') as f: graph_def = tf.GraphDef() graph_def.ParseFromString(f.read()) bottleneck_tensor, jpeg_data_tensor = tf.import_graph_def( graph_def, return_elements=[BOTTLENECK_TENSOR_NAME, JPEG_DATA_TENSOR_NAME]) bottleneck_input = tf.placeholder( tf.float32, [None, BOTTLENECK_TENSOR_SIZE], name='BottleneckInputPlaceholder') # 特征化过后的欲训练数据 ground_truth_input = tf.placeholder(tf.float32, [None, n_classes], name='GroundTruthInput') # 对应的真实结果 # 定义正向传播 with tf.name_scope('final_training_ops'): weights1 = tf.Variable( tf.truncated_normal([BOTTLENECK_TENSOR_SIZE, 1024], stddev=0.001)) biases1 = tf.Variable(tf.zeros(1024)) y1 = tf.matmul(bottleneck_input, weights1) + biases1 logits1 = tf.nn.sigmoid(y1) weights2 = tf.Variable( tf.truncated_normal([1024, n_classes], stddev=0.001)) biases2 = tf.Variable(tf.zeros(n_classes)) logits = tf.matmul(logits1, weights2) + biases2 final_tensor = tf.nn.softmax(logits) confusion_matrix = tf.confusion_matrix(tf.argmax(final_tensor, 1), tf.argmax(ground_truth_input, 1), num_classes=n_classes) # 定义损失函数 cross_entropy = tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits) cross_entropy_mean = tf.reduce_mean(cross_entropy) # 定义反向传播 train_step = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize( cross_entropy_mean) # 计算正确率 with tf.name_scope('evaluation'): correct_prediction = tf.equal(tf.argmax(final_tensor, 1), tf.argmax(ground_truth_input, 1)) evaluation_step = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) tf.summary.scalar("predict", evaluation_step) # 图定义结束 merged_summary_op = tf.summary.merge_all() with tf.Session() as sess: train_writer = tf.summary.FileWriter("logs/sigmoid_qyxx2/train", sess.graph) vali_writer = tf.summary.FileWriter("logs/sigmoid_qyxx2/vali", sess.graph) init = tf.initialize_all_variables() sess.run(init) for i in range(STEPS): # 每次获取一个batch的训练数据 try: train_bottlenecks, train_ground_truth = get_random_cached_bottlenecks( sess, n_classes, image_lists, BATCH, 'training', jpeg_data_tensor, bottleneck_tensor) except Exception as e: print("error") continue _, merged_summary_op_train = sess.run( [train_step, merged_summary_op], feed_dict={ bottleneck_input: train_bottlenecks, ground_truth_input: train_ground_truth }) # 在验证数据上测试正确率 if i % 50 == 0 or i + 1 == STEPS: try: validation_bottlenecks, validation_ground_truth = get_random_cached_bottlenecks( sess, n_classes, image_lists, BATCH * 50, 'validation', jpeg_data_tensor, bottleneck_tensor) except Exception as e: print("error vali") continue validation_accuracy, cm, merged_summary_op_vali = sess.run( [evaluation_step, confusion_matrix, merged_summary_op], feed_dict={ bottleneck_input: validation_bottlenecks, ground_truth_input: validation_ground_truth }) # draw_cm(cm) print(f"{i} validation_accuracy: {validation_accuracy}") train_writer.add_summary(merged_summary_op_train, i) vali_writer.add_summary(merged_summary_op_vali, i) # 在最后的测试数据上测试正确率 test_bottlenecks, test_ground_truth = get_test_bottlenecks( sess, image_lists, n_classes, jpeg_data_tensor, bottleneck_tensor) test_accuracy = sess.run(evaluation_step, feed_dict={ bottleneck_input: test_bottlenecks, ground_truth_input: test_ground_truth }) print(f"test_accuracy: {test_accuracy}")
def graph(self, in_training=True, submission=False): shape_name = lambda tensor: print("Tensor {0} has shape = {1}.\n". format(tensor.name, tensor.get_shape().as_list())) with tf.name_scope('input'): self.loader = load_data.Data_loader(in_training=in_training, in_size=self.in_size, batch_size=self.batch_size, n_epochs=self.n_epochs, aug_flip=self.aug_flip, submission=submission) data = self.loader.get_data() self.n_classes = self.loader.le.classes_.size self.path, self.X, self.Y = data if self.Y is None: self.Y = tf.placeholder(tf.int32, [ None, ]) self.X_reshape = tf.reshape(self.X, shape=[-1, self.in_h, self.in_w, 1]) for layer_no in range(1, self.n_layers + 1): if layer_no == 1: in_tensor = self.X_reshape n_filters = self.n_filters else: in_tensor = out_tensor n_filters += self.n_filters_inc conv = tf.layers.conv2d( inputs=in_tensor, filters=n_filters, kernel_size=[self.filter_size, self.filter_size], padding="same", activation=tf.nn.relu, name="conv_{0}".format(layer_no)) conv_bn = tf.layers.batch_normalization( conv, training=in_training, name="bn_{0}".format(layer_no)) pool = tf.layers.max_pooling2d( inputs=conv_bn, pool_size=[self.pool_size, self.pool_size], strides=2, name="pool_{0}".format(layer_no)) [shape_name(tensor) for tensor in [conv, conv_bn, pool]] out_tensor = pool shape = out_tensor.get_shape().as_list() print("Shape at lowest point = {0}".format(shape)) flat = tf.reshape(out_tensor, [-1, shape[1] * shape[2] * shape[3]]) #flat = tf.layers.dropout(flat, rate=0.05, training=in_training) dense = tf.layers.dense(inputs=flat, units=256, activation=tf.nn.relu) dense = tf.layers.batch_normalization(dense, training=in_training) self.logits = tf.layers.dense(inputs=dense, units=self.n_classes, activation=tf.nn.relu) self.softmax = tf.nn.softmax(self.logits, name="softmax_tensor") self.loss = tf.losses.sparse_softmax_cross_entropy(labels=self.Y, logits=self.logits) predictions = tf.argmax(input=self.logits, axis=1) with tf.variable_scope("cm"): n_classes = self.loader.le.classes_.size cm_diff = tf.confusion_matrix(labels=self.Y, predictions=predictions, num_classes=n_classes) self.cm_init = tf.get_variable("confusion_matrix", [n_classes, n_classes], dtype=tf.int32, initializer=tf.zeros_initializer()) self.cm = tf.assign_add(self.cm_init, cm_diff) correct_prediction = tf.equal(tf.cast(self.Y, tf.int64), predictions) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) self.metrics = { "predictions": predictions, "probabilities": self.softmax, "cm": self.cm, "acc": accuracy } self.global_step = tf.Variable(0, name='global_step', trainable=False) self.optimizer = tf.train.RMSPropOptimizer( learning_rate=self.learning_rate) extra_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(extra_ops): #for BN self.train_op = self.optimizer.minimize( loss=self.loss, global_step=self.global_step) self.saver = tf.train.Saver() total_params = np.sum([ np.prod(v.get_shape().as_list()) for v in tf.trainable_variables() ]) print("Number of trainable parameters = {0}.".format(total_params)) tf.summary.scalar('acc', self.metrics['acc']) tf.summary.scalar('loss', self.loss) self.merged = tf.summary.merge_all()
def _construct_graph(self): if self.random_state: tf.set_random_seed(self.random_state) np.random.seed(self.random_state) if self._initializer is None: if self.initializer == 'he': self._initializer = tf.contrib.layers.variance_scaling_initializer( ) if self._activation is None: if self.activation == 'relu': self._activation = tf.nn.relu if self._optimizer is None: if self.optimizer == 'adam': self._optimizer = tf.train.AdamOptimizer self.mlp_layers = len(self.num_neurons) - 2 X_heads = tf.placeholder(dtype=tf.float32, shape=[None, None, None, self.embedding_size], name="X_heads") X_bodies = tf.placeholder( dtype=tf.float32, shape=[None, None, None, self.embedding_size], name="X_bodies") X_head_sizes = tf.placeholder(dtype=tf.int32, shape=[None], name="head_sizes") X_body_sizes = tf.placeholder(dtype=tf.int32, shape=[None], name="body_sizes") X_head_sent_sizes = tf.placeholder(dtype=tf.int32, shape=[None, None], name="head_sent_sizes") X_body_sent_sizes = tf.placeholder(dtype=tf.int32, shape=[None, None], name="body_sent_sizes") # X_addtional = tf.placeholder(tf.float32,shape=[None,features_length],name="additonal_features") y_ = tf.placeholder(tf.int32, shape=[None], name="y") y_one_hot = tf.one_hot(y_, self.n_outputs, on_value=1.0, off_value=0.0, axis=-1, dtype=tf.float32) if self.dropout_rate: self._training = tf.placeholder_with_default(False, shape=[], name="training") self.keep_prob = tf.cond( self._training, lambda: tf.constant(1 - self.dropout_rate), lambda: tf.constant(1.0)) else: self._training = None pre_output = self._ann(X_heads, X_bodies, X_head_sizes, X_body_sizes, X_head_sent_sizes, X_body_sent_sizes) logits = tf.layers.dense(pre_output, self.n_outputs, kernel_initializer=self._initializer, name="logits") probabilities = tf.nn.softmax(logits, name="probabilities") if self.pos_weight is None: xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits( labels=y_, logits=logits) else: # self.logger.debug("weighted xentropy") xentropy = tf.nn.weighted_cross_entropy_with_logits( y_one_hot, logits, self.pos_weight) loss = tf.reduce_mean(xentropy, name="loss") variables = tf.trainable_variables() for v in variables: # self.logger.debug(v.name) self.logger.debug(self.name + ": " + v.name) # l2_loss = tf.add_n( # [tf.nn.l2_loss(v) for v in variables if 'bias' not in v.name and 'Variable' not in v.name]) * self.l2_lambda # loss += l2_loss optimizer = self._optimizer(learning_rate=self.learning_rate) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_op = optimizer.minimize(loss) correct = tf.nn.in_top_k(logits, y_, 1) accuracy = tf.reduce_mean(tf.cast(correct, tf.float32), name="accuracy") _, predicts = tf.nn.top_k(logits, k=1, sorted=False) confusion_matrix = tf.confusion_matrix(y_, predicts, num_classes=self.n_outputs, name="confusion_matrix") init = tf.global_variables_initializer() saver = tf.train.Saver( tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)) if self.tensorboard_logdir: now = datetime.utcnow().strftime('%Y%m%d-%H%M%S') tb_logdir = self.tensorboard_logdir + "/run{}".format(now) cost_summary = tf.summary.scalar("validation_loss", loss) acc_summary = tf.summary.scalar("validation_accuracy", accuracy) merged_summary = tf.summary.merge_all() file_writer = tf.summary.FileWriter(tb_logdir, tf.get_default_graph()) self._merged_summary = merged_summary self._file_writer = file_writer self._X_head, self._X_body, self._X_h_sizes, self._X_b_sizes, self._X_h_sent_sizes, self._X_b_sent_sizes, self.y = X_heads, X_bodies, X_head_sizes, X_body_sizes, X_head_sent_sizes, X_body_sent_sizes, y_ self._logits = logits self._probabilities = probabilities self._loss = loss self._training_op = training_op self._accuracy = accuracy self._confusion_matrix = confusion_matrix self._init, self._saver = init, saver
'Hb': tf.constant([], dtype=tf.float32) } return tf.estimator.export.ServingInputReceiver(inputs, inputs) classifier = tf.estimator.DNNClassifier( feature_columns=feature_columns, hidden_units=[3], n_classes=2) classifier.train( input_fn=lambda: my_input_fn(dfile, True, 8)) evaluate_result = classifier.evaluate( input_fn=lambda: my_input_fn(dfile, False, 4)) print("evaluate results") for key in evaluate_result: print(" {}, was: {}".format(key, evaluate_result[key])) predict_results = classifier.predict( input_fn=lambda: my_input_fn(dfile, False, 1)) print("Predictions") predictions = [] for prediction in predict_results: predictions.append(prediction["class_ids"][0]) confusion_matrix = tf.confusion_matrix(pctes["Clase"].tolist(), predictions) with tf.Session(): print('Confusion Matrix: \n\n', tf.Tensor.eval(confusion_matrix,feed_dict=None, session=None))
def main(_): best_acc = 0 best_step = 0 best_acc_istrain = 0 best_step_istrain = 0 # We want to see all the logging messages for this tutorial. tf.logging.set_verbosity(tf.logging.INFO) # Start a new TensorFlow session. sess = tf.InteractiveSession() # Begin by making sure we have the training data we need. If you already have # training data of your own, use `--data_url= ` on the command line to avoid # downloading. model_settings = models.prepare_model_settings( len(new_features_input.prepare_words_list_my(FLAGS.wanted_words.split(','))), FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms, FLAGS.window_stride_ms, FLAGS.dct_coefficient_count) audio_processor = new_features_input.AudioProcessor( FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.wanted_words.split(','), FLAGS.validation_percentage, FLAGS.testing_percentage) fingerprint_size = model_settings['fingerprint_size'] label_count = model_settings['label_count'] training_steps_list = list(map(int, FLAGS.how_many_training_steps.split(','))) learning_rates_list = list(map(float, FLAGS.learning_rate.split(','))) if len(training_steps_list) != len(learning_rates_list): raise Exception( '--how_many_training_steps and --learning_rate must be equal length ' 'lists, but are %d and %d long instead' % (len(training_steps_list), len(learning_rates_list))) ############################################## ############tensorflow modules########## fingerprint_input = tf.placeholder( tf.float32, [None, fingerprint_size], name='fingerprint_input') # ############ 模型创建 ########## istrain = tf.placeholder(tf.bool, name='istrain') logits,conv1,conv1bn,dsc1,dsc1bn,dsc1pw,dsc1pwbn,dsc2,dsc2bn,dsc2pw,dsc2pwbn,dsc3,dsc3bn,dsc3pw,dsc3pwbn,\ dsc4,dsc4bn,dsc4pw,dsc4pwbn,fc= models.create_model( fingerprint_input, model_settings, FLAGS.model_architecture, is_training=istrain) ############ 模型创建 ########## # logits, dropout_prob= models.create_model( # fingerprint_input, # model_settings, # FLAGS.model_architecture, # is_training=True) # Define loss and optimizer ############ 真实值 ########## ground_truth_input = tf.placeholder( tf.float32, [None, label_count], name='groundtruth_input') # Optionally we can add runtime checks to spot when NaNs or other symptoms of # numerical errors start occurring during training. control_dependencies = [] if FLAGS.check_nans: checks = tf.add_check_numerics_ops() control_dependencies = [checks] # Create the back propagation and training evaluation machinery in the graph. ############ 交叉熵计算 ########## # with tf.name_scope('cross_entropy'): # cross_entropy_mean = tf.reduce_mean( # tf.nn.softmax_cross_entropy_with_logits( # labels=ground_truth_input, logits=logits)) + beta*loss_norm with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits)) tf.summary.scalar('cross_entropy', cross_entropy_mean) ############ 学习率、准确率、混淆矩阵 ########## # learning_rate_input 学习率输入(tf.placeholder) # train_step 训练过程 (优化器) # predicted_indices 预测输出索引 # expected_indices 实际希望输出索引 # correct_prediction 正确预测矩阵 # confusion_matrix 混淆矩阵 # evaluation_step 正确分类概率(每个阶段) # global_step 全局训练阶段 # increment_global_step 全局训练阶段递增 learning_rate_input = tf.placeholder( tf.float32, [], name='learning_rate_input') update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_step = tf.train.AdamOptimizer( learning_rate_input).minimize(cross_entropy_mean) # with tf.name_scope('train'), tf.control_dependencies(control_dependencies): # learning_rate_input = tf.placeholder( # tf.float32, [], name='learning_rate_input') # # train_step = tf.train.GradientDescentOptimizer( # # learning_rate_input).minimize(cross_entropy_mean) # with tf.control_dependencies(update_ops): # train_step = tf.train.AdamOptimizer( # learning_rate_input).minimize(cross_entropy_mean) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix( expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) acc = tf.summary.scalar('accuracy', evaluation_step) global_step = tf.train.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) saver = tf.train.Saver(tf.global_variables(),max_to_keep=None)# max keep file // moren 5 # Merge all the summaries and write them out to /tmp/retrain_logs (by default) merged_summaries = tf.summary.merge_all() validation_merged_summaries = tf.summary.merge([tf.get_collection(tf.GraphKeys.SUMMARIES,'accuracy'),tf.get_collection(tf.GraphKeys.SUMMARIES,'cross_entropy')]) test_summaries = tf.summary.merge([acc]) test_summaries_istrain = tf.summary.merge([tf.get_collection(tf.GraphKeys.SUMMARIES,'accuracy'),tf.get_collection(tf.GraphKeys.SUMMARIES,'cross_entropy')]) #test_summaries_istrain = tf.summary.merge([acc]) train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', sess.graph) # validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation') test_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/test') test_istrain_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/test_istrain') tf.global_variables_initializer().run() start_step = 1 if FLAGS.start_checkpoint: models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) start_step = global_step.eval(session=sess) tf.logging.info('Training from step: %d ', start_step) # Save graph.pbtxt. tf.train.write_graph(sess.graph_def, FLAGS.train_dir, FLAGS.model_architecture + '.pbtxt') # Save list of words. with gfile.GFile( os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '_labels.txt'), 'w') as f: f.write('\n'.join(audio_processor.words_list)) ### # model1: fc # model2: conv :940k个parameter # model3:low_latancy_conv:~~model1 # model4: 750k # Training loop. ############################################# ######## 主循环 ###### ############################################# training_steps_max = np.sum(training_steps_list) for training_step in xrange(start_step, training_steps_max + 1): # Figure out what the current learning rate is. ####### 自动切换学习率 ####### if training_step <12000+1: learning_rate_value = learning_rates_list[0]*0.02**(training_step/12000) else: learning_rate_value = learning_rates_list[0]*0.02 #0.015 12000 training_steps_sum = 0 # for i in range(len(training_steps_list)): # training_steps_sum += training_steps_list[i] # if training_step <= training_steps_sum: # learning_rate_value = learning_rates_list[i] # break # Pull the audio samples we'll use for training. ####### audio处理器导入数据 ################################## ##get_data(self, how_many, offset, model_settings, background_frequency, ## background_volume_range, time_shift, mode, sess) ######################################################################## train_fingerprints, train_ground_truth = audio_processor.get_data_my( FLAGS.batch_size, 0, model_settings ,'training') #mid = np.abs(np.max(train_fingerprints) + np.min(train_fingerprints)) / 2 #half = np.max(train_fingerprints) - np.min(train_fingerprints) # train_fingerprints = ((train_fingerprints + mid) / half * 255).astype(int) train_fingerprints_mix, train_ground_truth_mix = mixup_data(train_fingerprints, train_ground_truth, 1) train_fingerprints = np.append(train_fingerprints, train_fingerprints_mix, axis=0) train_ground_truth = np.append(train_ground_truth, train_ground_truth_mix, axis=0) random_index = list(np.arange(FLAGS.batch_size*2)) np.random.shuffle(random_index) train_fingerprints = train_fingerprints[random_index, :] train_ground_truth = train_ground_truth[random_index, :] train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run( [ merged_summaries, evaluation_step, cross_entropy_mean, train_step, increment_global_step ], feed_dict={ fingerprint_input: train_fingerprints, ground_truth_input: train_ground_truth, learning_rate_input: learning_rate_value, istrain:True }) train_writer.add_summary(train_summary, training_step) tf.logging.info('Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' % (training_step, learning_rate_value, train_accuracy * 100, cross_entropy_value)) is_last_step = (training_step == training_steps_max) if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step: ############################################# ######## 测试集重复计算正确率和混淆矩阵 ###### set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) test_fingerprints, test_ground_truth = audio_processor.get_data_my( -1, 0, model_settings,'testing') #mid = np.abs(np.max(test_fingerprints) + np.min(test_fingerprints)) / 2 #half = np.max(test_fingerprints) - np.min(test_fingerprints) #test_fingerprints = ((test_fingerprints + mid) / half * 255).astype(int) final_summary,test_accuracy, conf_matrix = sess.run( [test_summaries,evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, istrain : False }) final_summary_istrain,test_accuracy_istrain= sess.run( [test_summaries_istrain,evaluation_step], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, istrain : True }) if test_accuracy > best_acc: best_acc = test_accuracy best_step = training_step if test_accuracy_istrain > best_acc_istrain: best_acc_istrain = test_accuracy_istrain best_step_istrain = training_step test_writer.add_summary(final_summary, training_step) test_istrain_writer.add_summary(final_summary_istrain, training_step) tf.logging.info('Confusion Matrix:\n %s' % (conf_matrix)) tf.logging.info('test accuracy = %.1f%% (N=%d)' % (test_accuracy * 100,6882)) tf.logging.info('test_istrain accuracy = %.1f%% (N=%d)' % (test_accuracy_istrain * 100,6882)) tf.logging.info('Best test accuracy before now = %.1f%% (N=%d)' % (best_acc * 100,6882) + ' at step of ' + str(best_step)) tf.logging.info('Best test_istrain accuracy before now = %.1f%% (N=%d)' % (best_acc_istrain * 100,6882) + ' at step of ' + str(best_step_istrain)) # Save the model checkpoint periodically. if (training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max): checkpoint_path = os.path.join(FLAGS.train_dir + '/'+FLAGS.model_architecture, FLAGS.model_architecture + '.ckpt') tf.logging.info('Saving to "%s-%d"', checkpoint_path, training_step) saver.save(sess, checkpoint_path, global_step=training_step) print_line = 'Best test accuracy before now = %.1f%% (N=%d)' % (best_acc * 100,6882) + ' at step of ' + str(best_step) + '\n' + \ 'Best test_istrain accuracy before now = %.1f%% (N=%d)' % (best_acc_istrain * 100,6882) + ' at step of ' + str(best_step_istrain) if training_step == training_steps_max: with open(FLAGS.train_dir + '/' +FLAGS.model_architecture+ '/details.txt', 'w') as f: f.write(print_line)
# optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) optimizer = tf.train.AdagradOptimizer(learning_rate=learning_rate, initial_accumulator_value=0.1, name="optimizer_op") # optimizer = tf.train.GradientDescentOptimizer(learning_rate=learning_rate) train_op = optimizer.minimize(loss_op, name="train_op") # Evaluate model (with test logits, for dropout to be disabled) correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1), name="correct_prediction") accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32), name="accuracy") # define confusion matrix conf_matrix = tf.confusion_matrix(tf.argmax(Y, 1), tf.argmax(prediction, 1), num_classes, name="confusion_matrix") # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() # run training and testing experiment conf = tf.ConfigProto() conf.gpu_options.allow_growth = True with tf.Session(config=conf) as sess: # debug # sess=tf_debug.LocalCLIDebugWrapperSession(sess) # sess.add_tensor_filter("has_inf_or_nan",tf_debug.has_inf_or_nan) # Run the initializer
########################## softmax = tf.nn.softmax(logits, name='labels_softmax') print(softmax) with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=ground_truth_input, logits=logits)) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #********************************************************************* print(" ***************** audio processor ********************") training_datas = len(audio_processor.data_index['training']) + len( audio_processor.unknown_index['training']) validation_datas = len(audio_processor.data_index['validation']) + len( audio_processor.unknown_index['validation']) testing_datas = len(audio_processor.data_index['testing']) + len( audio_processor.unknown_index['testing']) print("* total samples : " + str(training_datas + validation_datas + testing_datas)) print("* training samples : "+str(len(audio_processor.data_index['training'])) + ' + ' \
shape_batch, model_settings, model_settings["model_architecture"], is_training=False) # Define loss with tf.name_scope('cross_entropy'): loss = tf.losses.sparse_softmax_cross_entropy(labels=label_batch, logits=logits) tf.summary.scalar('cross_entropy', loss) # Define evaluation metrics predicted_indices = tf.argmax(logits, 1) correct_prediction = tf.equal(predicted_indices, label_batch) confusion_matrix = tf.confusion_matrix( label_batch, predicted_indices, num_classes=model_settings["num_classes"]) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # Merge all the summaries merged_summaries = tf.summary.merge_all() # Create a session for running operations in the Graph. with tf.Session(config=tf.ConfigProto(log_device_placement=False, allow_soft_placement=True, gpu_options=tf.GPUOptions( allow_growth=True))) as sess: # restore the graph saver = tf.train.Saver(tf.global_variables(), max_to_keep=30) ckpt_path = os.path.dirname(FLAGS.config_dir)
def main(_): # We want to see all the logging messages for this tutorial. tf.logging.set_verbosity(tf.logging.INFO) # Start a new TensorFlow session. sess = tf.InteractiveSession() # Begin by making sure we have the training data we need. If you already have # training data of your own, use `--data_url= ` on the command line to avoid # downloading. model_settings = models.prepare_model_settings( len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))), FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms, FLAGS.window_stride_ms, FLAGS.dct_coefficient_count) audio_processor = input_data.AudioProcessor( FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage, FLAGS.wanted_words.split(','), FLAGS.validation_percentage, FLAGS.testing_percentage, model_settings) fingerprint_size = model_settings['fingerprint_size'] label_count = model_settings['label_count'] time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000) # Figure out the learning rates for each training phase. Since it's often # effective to have high learning rates at the start of training, followed by # lower levels towards the end, the number of steps and learning rates can be # specified as comma-separated lists to define the rate at each stage. For # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001 # will run 13,000 training loops in total, with a rate of 0.001 for the first # 10,000, and 0.0001 for the final 3,000. training_steps_list = list(map(int, FLAGS.how_many_training_steps.split(','))) learning_rates_list = list(map(float, FLAGS.learning_rate.split(','))) if len(training_steps_list) != len(learning_rates_list): raise Exception( '--how_many_training_steps and --learning_rate must be equal length ' 'lists, but are %d and %d long instead' % (len(training_steps_list), len(learning_rates_list))) fingerprint_input = tf.placeholder( tf.float32, [None, fingerprint_size], name='fingerprint_input') logits, dropout_prob = models.create_model( fingerprint_input, model_settings, FLAGS.model_architecture, is_training=True) # Define loss and optimizer ground_truth_input = tf.placeholder( tf.float32, [None, label_count], name='groundtruth_input') # Optionally we can add runtime checks to spot when NaNs or other symptoms of # numerical errors start occurring during training. control_dependencies = [] if FLAGS.check_nans: checks = tf.add_check_numerics_ops() control_dependencies = [checks] # Create the back propagation and training evaluation machinery in the graph. with tf.name_scope('cross_entropy'): cross_entropy_mean = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( labels=ground_truth_input, logits=logits)) tf.summary.scalar('cross_entropy', cross_entropy_mean) with tf.name_scope('train'), tf.control_dependencies(control_dependencies): learning_rate_input = tf.placeholder( tf.float32, [], name='learning_rate_input') train_step = tf.train.GradientDescentOptimizer( learning_rate_input).minimize(cross_entropy_mean) predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices, num_classes=label_count) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', evaluation_step) global_step = tf.train.get_or_create_global_step() increment_global_step = tf.assign(global_step, global_step + 1) saver = tf.train.Saver(tf.global_variables()) # Merge all the summaries and write them out to /tmp/retrain_logs (by default) merged_summaries = tf.summary.merge_all() train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', sess.graph) validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation') tf.global_variables_initializer().run() start_step = 1 if FLAGS.start_checkpoint: models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) start_step = global_step.eval(session=sess) tf.logging.info('Training from step: %d ', start_step) # Save graph.pbtxt. tf.train.write_graph(sess.graph_def, FLAGS.train_dir, FLAGS.model_architecture + '.pbtxt') # Save list of words. with gfile.GFile( os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '_labels.txt'), 'w') as f: f.write('\n'.join(audio_processor.words_list)) # Training loop. training_steps_max = np.sum(training_steps_list) for training_step in xrange(start_step, training_steps_max + 1): # Figure out what the current learning rate is. training_steps_sum = 0 for i in range(len(training_steps_list)): training_steps_sum += training_steps_list[i] if training_step <= training_steps_sum: learning_rate_value = learning_rates_list[i] break # Pull the audio samples we'll use for training. train_fingerprints, train_ground_truth = audio_processor.get_data( FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency, FLAGS.background_volume, time_shift_samples, 'training', sess) # Run the graph with this batch of training data. train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run( [ merged_summaries, evaluation_step, cross_entropy_mean, train_step, increment_global_step ], feed_dict={ fingerprint_input: train_fingerprints, ground_truth_input: train_ground_truth, learning_rate_input: learning_rate_value, dropout_prob: 0.5 }) train_writer.add_summary(train_summary, training_step) tf.logging.info('Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' % (training_step, learning_rate_value, train_accuracy * 100, cross_entropy_value)) is_last_step = (training_step == training_steps_max) if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step: set_size = audio_processor.set_size('validation') total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): validation_fingerprints, validation_ground_truth = ( audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'validation', sess)) # Run a validation step and capture training summaries for TensorBoard # with the `merged` op. validation_summary, validation_accuracy, conf_matrix = sess.run( [merged_summaries, evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: validation_fingerprints, ground_truth_input: validation_ground_truth, dropout_prob: 1.0 }) validation_writer.add_summary(validation_summary, training_step) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (validation_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' % (training_step, total_accuracy * 100, set_size)) # Save the model checkpoint periodically. if (training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max): checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '.ckpt') tf.logging.info('Saving to "%s-%d"', checkpoint_path, training_step) saver.save(sess, checkpoint_path, global_step=training_step) set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) total_accuracy = 0 total_conf_matrix = None for i in xrange(0, set_size, FLAGS.batch_size): test_fingerprints, test_ground_truth = audio_processor.get_data( FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess) test_accuracy, conf_matrix = sess.run( [evaluation_step, confusion_matrix], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth, dropout_prob: 1.0 }) batch_size = min(FLAGS.batch_size, set_size - i) total_accuracy += (test_accuracy * batch_size) / set_size if total_conf_matrix is None: total_conf_matrix = conf_matrix else: total_conf_matrix += conf_matrix tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) tf.logging.info('Final test accuracy = %.1f%% (N=%d)' % (total_accuracy * 100, set_size))
def main(): parser = create_parser() argcomplete.autocomplete(parser) args = parser.parse_args() print_outputs = args.print_outputs sess = tf.InteractiveSession() model_settings = models.prepare_model_settings( len(input_data.prepare_words_list(args.wanted_words.split(','))), args.sample_rate, args.clip_duration_ms, args.window_size_ms, args.window_stride_ms, args.dct_coefficient_count) audio_processor = input_data.AudioProcessor(args.data_url, args.data_dir, args.silence_percentage, args.unknown_percentage, args.wanted_words.split(','), args.validation_percentage, args.testing_percentage, model_settings) label_count = model_settings['label_count'] fingerprint_size = model_settings['fingerprint_size'] fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size], name='fingerprint_input') if print_outputs: logits, first_conv_val, first_weights, first_bias, second_weights, second_bias, third_weights, second_conv_val = models.create_model( fingerprint_input, model_settings, args.model_architecture, is_training=False, print_outputs=True) else: logits = models.create_model(fingerprint_input, model_settings, args.model_architecture, is_training=False, print_outputs=False) # load weights/biases from checkpoint models.load_variables_from_checkpoint(sess, args.start_checkpoint) # Define loss and optimizer ground_truth_input = tf.placeholder(tf.float32, [None, label_count], name='groundtruth_input') predicted_indices = tf.argmax(logits, 1) expected_indices = tf.argmax(ground_truth_input, 1) correct_prediction = tf.equal(predicted_indices, expected_indices) confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices) evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('accuracy', evaluation_step) #generate test outputs set_size = audio_processor.set_size('testing') tf.logging.info('set_size=%d', set_size) batch_size = args.batch_size directory = args.directory test_fingerprints, test_ground_truth = audio_processor.get_data( batch_size, 0, model_settings, 0.0, 0.0, 0, 'testing', sess, True) if print_outputs: outfc, outconv1, weights1, bias1, weights2, bias2, weights3, outconv2, test_accuracy, expected, predicted = sess.run( [ logits, first_conv_val, first_weights, first_bias, second_weights, second_bias, third_weights, second_conv_val, evaluation_step, expected_indices, predicted_indices ], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth #dropout_prob: 1.0 }) else: outfc, test_accuracy, expected, predicted = sess.run( [logits, evaluation_step, expected_indices, predicted_indices], feed_dict={ fingerprint_input: test_fingerprints, ground_truth_input: test_ground_truth #dropout_prob: 1.0 }) print("expected/predicted") print(expected) print(predicted) # print image in a .h file, to be include for img_num in range(batch_size): in_feat = np.array(98 * 40) #print(test_fingerprints[img_num]*64) in_feat = np.reshape(test_fingerprints[img_num] * 64 + 0.5, (98 * 40)) for i in range(0, 40 * 98): in_feat[i] = math.floor(in_feat[i]) in_feat_int = np.array(98 * 40) in_feat_int = in_feat.astype(int) format = '%d' np.savetxt("./data/in_feat_{}_{}.txt".format(img_num, expected[img_num]), in_feat_int, delimiter=", ", newline=",\n", fmt=format) if print_outputs: #outconv1_2D = np.reshape(outconv1,(batch_size*32,79*33)) outconv1_2D = np.reshape(outconv1, (batch_size * 32, 39 * 16)) first_weights_2D = np.reshape(weights1, (32, 8 * 20)) weights2 = np.reshape(weights2, (10 * 4, 32, 32)) second_weights_2D = np.reshape(weights2.transpose(), (32 * 32, 4 * 10)) weights3 = np.reshape(weights3, (13 * 30, 32, 12)) print("SHAPE WEIGHTS3") print(np.shape(weights3)) third_weights_2D = np.reshape(weights3.transpose(), (12, 32 * 13 * 30)) outconv2_2D = np.reshape(outconv2, (batch_size * 32, 30 * 13)) print("SHAPE WEIGHTS2") print(np.shape(weights2)) np.savetxt("./data/outFC.txt", outfc, delimiter=",") np.savetxt("./data/outConv1.txt", outconv1_2D, delimiter=",") np.savetxt("./data/weights1.txt", first_weights_2D, delimiter=",") np.savetxt("./data/bias1.txt", bias1, delimiter=",") np.savetxt("./data/weights2.txt", second_weights_2D * 1024 * 32, delimiter=",") np.savetxt("./data/bias2.txt", bias2, delimiter=",") np.savetxt("./data/outConv2.txt", outconv2_2D, delimiter=",") tf.logging.info('test accuracy = %.1f%% (N=%d)' % (test_accuracy, batch_size)) np.savetxt("./data/weights3.txt", third_weights_2D * 1024 * 32, delimiter=",\n") # dump file in a 40*98 pgm image with 16bits pixels s_16b = np.array([40 * 98], dtype=np.uint16) #shift left by 7 bits for i in range(batch_size): s_16b = test_fingerprints[i] * 64 + 0.5 #print(s_16b) with open( os.path.join(directory, "features_{}_{}.pgm".format(i, expected[i])), 'wb') as f: hdr = 'P5' + '\n' + str(40) + ' ' + str(98) + ' ' + str( 65335) + '\n' f.write(hdr.encode()) np.uint16(s_16b).tofile(f) print("finished: test accuracy = %.1f%%" % (test_accuracy * 100))