def save_bottleneck_features(self, inputs, file_path, batch_size=None, data_type=np.float32): inputs_shape = inputs.shape img_mode = 'L' if inputs_shape[3] == 1 else 'RGB' with open(file_path, 'wb') as f: if batch_size: batch_generator = utils.get_batches(inputs, batch_size=batch_size, keep_last=True) if len(inputs) % batch_size == 0: n_batch = len(inputs) // batch_size else: n_batch = len(inputs) // batch_size + 1 for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit='batch'): inputs_batch = next(batch_generator) inputs_batch = utils.imgs_scale_to_255( inputs_batch).astype(data_type) if inputs_shape[1:3] != (224, 224): inputs_batch = utils.img_resize( inputs_batch, (224, 224), img_mode=img_mode, resize_filter=Image.ANTIALIAS, verbose=False, ).astype(data_type) if inputs_shape[3] == 1: inputs_batch = np.concatenate( [inputs_batch, inputs_batch, inputs_batch], axis=-1) assert inputs_batch.shape[1:] == (224, 224, 3) bf_batch = self._extract_features(inputs_batch) f.write(pickle.dumps(bf_batch)) # Release memory K.clear_session() tf.reset_default_graph() else: inputs = utils.imgs_scale_to_255(inputs).astype(data_type) if inputs_shape[3] == 1: inputs = np.concatenate([inputs, inputs, inputs], axis=-1) assert inputs.shape[1:] == (224, 224, 3) bottleneck_features = self._extract_features(inputs) f.write(pickle.dumps(bottleneck_features))
def _eval_on_batches(self, sess, inputs, labels, loss, accuracy, clf_loss, rec_loss, rec_images, x, y, n_batch): """ Calculate losses and accuracies of full train set. """ loss_all = [] acc_all = [] clf_loss_all = [] rec_loss_all = [] step = 0 _batch_generator = utils.get_batches(x, y, self.cfg.TEST_BATCH_SIZE) if self.cfg.TEST_WITH_RECONSTRUCTION: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batches'): step += 1 x_batch, y_batch = next(_batch_generator) loss_i, clf_loss_i, rec_loss_i, acc_i = \ sess.run([loss, clf_loss, rec_loss, accuracy], feed_dict={inputs: x_batch, labels: y_batch}) loss_all.append(loss_i) clf_loss_all.append(clf_loss_i) rec_loss_all.append(rec_loss_i) acc_all.append(acc_i) # Save reconstruct images if self.cfg.TEST_SAVE_IMAGE_STEP is not None: if step % self.cfg.TEST_SAVE_IMAGE_STEP == 0: self._save_images(sess, rec_images, inputs, labels, x_batch, y_batch, step) clf_loss = sum(clf_loss_all) / len(clf_loss_all) rec_loss = sum(rec_loss_all) / len(rec_loss_all) else: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batches'): x_batch, y_batch = next(_batch_generator) loss_i, acc_i = \ sess.run([loss, accuracy], feed_dict={inputs: x_batch, labels: y_batch}) loss_all.append(loss_i) acc_all.append(acc_i) clf_loss, rec_loss = None, None loss = sum(loss_all) / len(loss_all) accuracy = sum(acc_all) / len(acc_all) return loss, clf_loss, rec_loss, accuracy
def get_bottleneck_features(self, inputs, batch_size=None, data_type=np.float32, pooling='avg'): # Check image size for transfer learning models inputs_shape = inputs.shape assert inputs_shape[1:3] == (224, 224) # Scale to 0-255 and extract features if batch_size: batch_generator = utils.get_batches( inputs, batch_size=batch_size, keep_last=True) n_batch = len(inputs) // batch_size + 1 bottleneck_features = [] for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit='batch'): inputs_batch = next(batch_generator) inputs_batch = utils.imgs_scale_to_255(inputs_batch).astype(data_type) if inputs_shape[3] == 1: inputs_batch = np.concatenate( [inputs_batch, inputs_batch, inputs_batch], axis=-1) assert inputs_batch.shape[1:] == (224, 224, 3) bf_batch = self._extract_features(inputs_batch, pooling=pooling) bottleneck_features.append(bf_batch) # Release memory K.clear_session() tf.reset_default_graph() bottleneck_features = np.concatenate(bottleneck_features, axis=0) else: inputs = utils.imgs_scale_to_255(inputs).astype(data_type) if inputs_shape[3] == 1: inputs = np.concatenate([inputs, inputs, inputs], axis=-1) bottleneck_features = self._extract_features(inputs, pooling=pooling) # Check data shape assert len(bottleneck_features) == len(inputs) assert bottleneck_features.shape[1:] == \ self._get_bottleneck_feature_shape(pooling=pooling) return bottleneck_features
def _get_preds_vector(self, sess, inputs, preds, is_training): """Get prediction vectors of full train set.""" utils.thin_line() print('Getting prediction vectors...') pred_all = [] _batch_generator = utils.get_batches( self.x_test, batch_size=self.cfg.TEST_BATCH_SIZE, keep_last=True) if len(self.x_test) % self.cfg.TEST_BATCH_SIZE == 0: n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE) else: n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE) + 1 for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batch'): x_batch = next(_batch_generator) # The last batch which has less examples len_batch = len(x_batch) if len_batch != self.cfg.TEST_BATCH_SIZE: for i in range(self.cfg.TEST_BATCH_SIZE - len_batch): x_batch = np.append(x_batch, np.expand_dims(np.zeros_like( x_batch[0]), axis=0), axis=0) assert len(x_batch) == self.cfg.TEST_BATCH_SIZE pred_i = sess.run(preds, feed_dict={ inputs: x_batch, is_training: False }) if len_batch != self.cfg.TEST_BATCH_SIZE: pred_i = pred_i[:len_batch] pred_all.extend(list(pred_i)) assert len(pred_all) == len(self.x_test), (len(pred_all), len(self.x_test)) return np.array(pred_all)
def _trainer(self, sess): utils.thick_line() print('Training...') # Merge all the summaries and create writers train_summary_path = join(self.summary_path, 'train') valid_summary_path = join(self.summary_path, 'valid') utils.check_dir([train_summary_path, valid_summary_path]) train_writer = tf.summary.FileWriter(train_summary_path, sess.graph) valid_writer = tf.summary.FileWriter(valid_summary_path) sess.run(tf.global_variables_initializer()) step = 0 for epoch_i in range(self.cfg.EPOCHS): epoch_start_time = time.time() utils.thick_line() print('Training on epoch: {}/{}'.format(epoch_i + 1, self.cfg.EPOCHS)) if self.cfg.DISPLAY_STEP is not None: for x_batch, y_batch in utils.get_batches( self.x_train, self.y_train, self.cfg.BATCH_SIZE): step += 1 # Training optimizer sess.run(self.optimizer, feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.step: step - 1, self.is_training: True }) # Display training information if step % self.cfg.DISPLAY_STEP == 0: self._display_status(sess, x_batch, y_batch, epoch_i, step) # Save training logs if self.cfg.SAVE_LOG_STEP is not None: if step % self.cfg.SAVE_LOG_STEP == 0: self._save_logs(sess, train_writer, valid_writer, x_batch, y_batch, epoch_i, step) # Save reconstruction images if self.cfg.SAVE_IMAGE_STEP is not None: if self.cfg.WITH_RECONSTRUCTION: if step % self.cfg.SAVE_IMAGE_STEP == 0: self._save_images(sess, self.train_image_path, x_batch, y_batch, step, epoch_i=epoch_i) # Save models if self.cfg.SAVE_MODEL_MODE == 'per_batch': if step % self.cfg.SAVE_MODEL_STEP == 0: self._save_model(sess, self.saver, step) # Evaluate on full set if self.cfg.FULL_SET_EVAL_MODE == 'per_batch': if step % self.cfg.FULL_SET_EVAL_STEP == 0: self._eval_on_full_set(sess, epoch_i, step) utils.thick_line() else: utils.thin_line() train_batch_generator = utils.get_batches( self.x_train, self.y_train, self.cfg.BATCH_SIZE) for _ in tqdm(range(self.n_batch_train), total=self.n_batch_train, ncols=100, unit=' batches'): step += 1 x_batch, y_batch = next(train_batch_generator) # Training optimizer sess.run(self.optimizer, feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.step: step - 1, self.is_training: True }) # Save training logs if self.cfg.SAVE_LOG_STEP is not None: if step % self.cfg.SAVE_LOG_STEP == 0: self._save_logs(sess, train_writer, valid_writer, x_batch, y_batch, epoch_i, step) # Save reconstruction images if self.cfg.SAVE_IMAGE_STEP is not None: if self.cfg.WITH_RECONSTRUCTION: if step % self.cfg.SAVE_IMAGE_STEP == 0: self._save_images(sess, self.train_image_path, x_batch, y_batch, step, silent=True, epoch_i=epoch_i) # Save models if self.cfg.SAVE_MODEL_MODE == 'per_batch': if step % self.cfg.SAVE_MODEL_STEP == 0: self._save_model(sess, self.saver, step, silent=True) # Evaluate on full set if self.cfg.FULL_SET_EVAL_MODE == 'per_batch': if step % self.cfg.FULL_SET_EVAL_STEP == 0: self._eval_on_full_set(sess, epoch_i, step, silent=True) if self.cfg.SAVE_MODEL_MODE == 'per_epoch': if (epoch_i + 1) % self.cfg.SAVE_MODEL_STEP == 0: self._save_model(sess, self.saver, epoch_i) if self.cfg.FULL_SET_EVAL_MODE == 'per_epoch': if (epoch_i + 1) % self.cfg.FULL_SET_EVAL_STEP == 0: self._eval_on_full_set(sess, epoch_i, step) utils.thin_line() print('Epoch done! Using time: {:.2f}'.format(time.time() - epoch_start_time)) utils.thick_line() print('Training finished! Using time: {:.2f}'.format(time.time() - self.start_time)) utils.thick_line() # Evaluate on test set after training if self.cfg.TEST_AFTER_TRAINING: self._test_after_training(sess) utils.thick_line() print('All task finished! Total time: {:.2f}'.format(time.time() - self.start_time)) utils.thick_line()
def _test_after_training(self, sess): """ Evaluate on the test set after training. """ test_start_time = time.time() utils.thick_line() print('Testing...') # Check directory of paths utils.check_dir([self.test_log_path]) if self.cfg.WITH_RECONSTRUCTION: if self.cfg.TEST_SAVE_IMAGE_STEP is not None: utils.check_dir([self.test_image_path]) # Load data utils.thin_line() print('Loading test set...') utils.thin_line() x_test = utils.load_data_from_pkl( join(self.preprocessed_path, 'x_test.p')) y_test = utils.load_data_from_pkl( join(self.preprocessed_path, 'y_test.p')) n_batch_test = len(y_test) // self.cfg.BATCH_SIZE utils.thin_line() print('Calculating loss and accuracy on test set...') loss_test_all = [] acc_test_all = [] clf_loss_test_all = [] rec_loss_test_all = [] step = 0 _test_batch_generator = utils.get_batches(x_test, y_test, self.cfg.BATCH_SIZE) if self.cfg.WITH_RECONSTRUCTION: for _ in tqdm(range(n_batch_test), total=n_batch_test, ncols=100, unit=' batches'): step += 1 test_batch_x, test_batch_y = next(_test_batch_generator) loss_test_i, clf_loss_i, rec_loss_i, acc_test_i = sess.run( [self.loss, self.clf_loss, self.rec_loss, self.accuracy], feed_dict={ self.inputs: test_batch_x, self.labels: test_batch_y, self.is_training: False }) loss_test_all.append(loss_test_i) acc_test_all.append(acc_test_i) clf_loss_test_all.append(clf_loss_i) rec_loss_test_all.append(rec_loss_i) # Save reconstruct images if self.cfg.TEST_SAVE_IMAGE_STEP is not None: if step % self.cfg.TEST_SAVE_IMAGE_STEP == 0: self._save_images(sess, self.test_image_path, test_batch_x, test_batch_y, step, silent=False) clf_loss_test = sum(clf_loss_test_all) / len(clf_loss_test_all) rec_loss_test = sum(rec_loss_test_all) / len(rec_loss_test_all) else: for _ in tqdm(range(n_batch_test), total=n_batch_test, ncols=100, unit=' batches'): test_batch_x, test_batch_y = next(_test_batch_generator) loss_test_i, acc_test_i = sess.run( [self.loss, self.accuracy], feed_dict={ self.inputs: test_batch_x, self.labels: test_batch_y, self.is_training: False }) loss_test_all.append(loss_test_i) acc_test_all.append(acc_test_i) clf_loss_test, rec_loss_test = None, None loss_test = sum(loss_test_all) / len(loss_test_all) acc_test = sum(acc_test_all) / len(acc_test_all) # Print losses and accuracy utils.thin_line() print('Test_Loss: {:.4f}\n'.format(loss_test), 'Test_Accuracy: {:.2f}%'.format(acc_test * 100)) if self.cfg.WITH_RECONSTRUCTION: utils.thin_line() print('Test_Train_Loss: {:.4f}\n'.format(clf_loss_test), 'Test_Reconstruction_Loss: {:.4f}'.format(rec_loss_test)) # Save test log utils.save_test_log(self.test_log_path, loss_test, acc_test, clf_loss_test, rec_loss_test, self.cfg.WITH_RECONSTRUCTION) utils.thin_line() print('Testing finished! Using time: {:.2f}'.format(time.time() - test_start_time))
def _eval_on_batches(self, mode, sess, x, y, n_batch, silent=False): """ Calculate losses and accuracies of full train set. """ loss_all = [] acc_all = [] clf_loss_all = [] rec_loss_all = [] if not silent: utils.thin_line() print( 'Calculating loss and accuracy of full {} set...'.format(mode)) _batch_generator = utils.get_batches(x, y, self.cfg.BATCH_SIZE) if self.cfg.WITH_RECONSTRUCTION: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batches'): x_batch, y_batch = next(_batch_generator) loss_i, clf_loss_i, rec_loss_i, acc_i = sess.run( [ self.loss, self.clf_loss, self.rec_loss, self.accuracy ], feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.is_training: False }) loss_all.append(loss_i) clf_loss_all.append(clf_loss_i) rec_loss_all.append(rec_loss_i) acc_all.append(acc_i) clf_loss = sum(clf_loss_all) / len(clf_loss_all) rec_loss = sum(rec_loss_all) / len(rec_loss_all) else: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batches'): x_batch, y_batch = next(_batch_generator) loss_i, acc_i = sess.run( [self.loss, self.accuracy], feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.is_training: False }) loss_all.append(loss_i) acc_all.append(acc_i) clf_loss, rec_loss = None, None else: if self.cfg.WITH_RECONSTRUCTION: for x_batch, y_batch in utils.get_batches( x, y, self.cfg.BATCH_SIZE): loss_i, clf_loss_i, rec_loss_i, acc_i = sess.run( [ self.loss, self.clf_loss, self.rec_loss, self.accuracy ], feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.is_training: False }) loss_all.append(loss_i) clf_loss_all.append(clf_loss_i) rec_loss_all.append(rec_loss_i) acc_all.append(acc_i) clf_loss = sum(clf_loss_all) / len(clf_loss_all) rec_loss = sum(rec_loss_all) / len(rec_loss_all) else: for x_batch, y_batch in utils.get_batches( x, y, self.cfg.BATCH_SIZE): loss_i, acc_i = sess.run( [self.loss, self.accuracy], feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.is_training: False }) loss_all.append(loss_i) acc_all.append(acc_i) clf_loss, rec_loss = None, None loss = sum(loss_all) / len(loss_all) accuracy = sum(acc_all) / len(acc_all) return loss, clf_loss, rec_loss, accuracy
def _eval_on_batches(self, sess, inputs, labels, input_imgs, is_training, preds, loss, acc, clf_loss, rec_loss, rec_imgs): """Calculate losses and accuracies of full train set.""" pred_all = [] loss_all = [] acc_all = [] clf_loss_all = [] rec_loss_all = [] step = 0 batch_generator = utils.get_batches( x=self.x_test, y=self.y_test, imgs=self.imgs_test, batch_size=self.cfg.TEST_BATCH_SIZE, keep_last=True) if len(self.x_test) % self.cfg.TEST_BATCH_SIZE == 0: n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE) else: n_batch = (len(self.x_test) // self.cfg.TEST_BATCH_SIZE) + 1 if self.cfg.TEST_WITH_REC: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batch'): step += 1 x_batch, y_batch, imgs_batch = next(batch_generator) len_batch = len(x_batch) if len_batch == self.cfg.TEST_BATCH_SIZE: pred_i, loss_i, clf_loss_i, rec_loss_i, acc_i = \ sess.run([preds, loss, clf_loss, rec_loss, acc], feed_dict={inputs: x_batch, labels: y_batch, input_imgs: imgs_batch, is_training: False}) loss_all.append(loss_i) clf_loss_all.append(clf_loss_i) rec_loss_all.append(rec_loss_i) acc_all.append(acc_i) # Save reconstruct images if self.cfg.TEST_SAVE_IMAGE_STEP: if step % self.cfg.TEST_SAVE_IMAGE_STEP == 0: self._save_images(sess, rec_imgs, inputs, labels, is_training, x_batch, y_batch, imgs_batch, step=step) else: # The last batch which has less examples for i in range(self.cfg.TEST_BATCH_SIZE - len_batch): x_batch = np.append(x_batch, np.expand_dims(np.zeros_like( x_batch[0]), axis=0), axis=0) assert len(x_batch) == self.cfg.TEST_BATCH_SIZE pred_i = sess.run(preds, feed_dict={ inputs: x_batch, is_training: False }) pred_i = pred_i[:len_batch] pred_all.extend(list(pred_i)) clf_loss_ = sum(clf_loss_all) / len(clf_loss_all) rec_loss_ = sum(rec_loss_all) / len(rec_loss_all) else: for _ in tqdm(range(n_batch), total=n_batch, ncols=100, unit=' batches'): x_batch, y_batch, imgs_batch = next(batch_generator) len_batch = len(x_batch) if len_batch == self.cfg.TEST_BATCH_SIZE: pred_i, loss_i, acc_i = \ sess.run([preds, loss, acc], feed_dict={inputs: x_batch, labels: y_batch, input_imgs: imgs_batch, is_training: False}) loss_all.append(loss_i) acc_all.append(acc_i) else: # The last batch which has less examples for i in range(self.cfg.TEST_BATCH_SIZE - len_batch): x_batch = np.append(x_batch, np.expand_dims(np.zeros_like( x_batch[0]), axis=0), axis=0) assert len(x_batch) == self.cfg.TEST_BATCH_SIZE pred_i = sess.run(preds, feed_dict={ inputs: x_batch, is_training: False }) pred_i = pred_i[:len_batch] pred_all.extend(list(pred_i)) clf_loss_, rec_loss_ = None, None loss_ = sum(loss_all) / len(loss_all) acc_ = sum(acc_all) / len(acc_all) assert len(pred_all) == len(self.x_test), (len(pred_all), len(self.x_test)) preds_vec = np.array(pred_all) return preds_vec, loss_, clf_loss_, rec_loss_, acc_
def _trainer(self, sess): utils.thick_line() print('Training...') # Merge all the summaries and create writers train_summary_path = join(self.summary_path, 'train') valid_summary_path = join(self.summary_path, 'valid') utils.check_dir([train_summary_path, valid_summary_path]) utils.thin_line() print('Generating TensorFLow summary writer...') train_writer = tf.summary.FileWriter(train_summary_path, sess.graph) valid_writer = tf.summary.FileWriter(valid_summary_path) sess.run(tf.global_variables_initializer()) step = 0 for epoch_i in range(self.cfg.EPOCHS): epoch_start_time = time.time() utils.thick_line() print('Training on epoch: {}/{}'.format(epoch_i + 1, self.cfg.EPOCHS)) utils.thin_line() train_batch_generator = utils.get_batches( x=self.x_train, y=self.y_train, imgs=self.imgs_train, batch_size=self.cfg.BATCH_SIZE) if self.cfg.DISPLAY_STEP: iterator = range(self.n_batch_train) silent = False else: iterator = tqdm(range(self.n_batch_train), total=self.n_batch_train, ncols=100, unit=' batch') silent = True for _ in iterator: step += 1 x_batch, y_batch, imgs_batch = next(train_batch_generator) # Training optimizer sess.run(self.optimizer, feed_dict={ self.inputs: x_batch, self.labels: y_batch, self.input_imgs: imgs_batch, self.step: step - 1, self.is_training: True }) # Display training information if self.cfg.DISPLAY_STEP: if step % self.cfg.DISPLAY_STEP == 0: self._display_status(sess, x_batch, y_batch, imgs_batch, epoch_i, step - 1) # Save training logs if self.cfg.SAVE_LOG_STEP: if step % self.cfg.SAVE_LOG_STEP == 0: self._save_logs(sess, train_writer, valid_writer, x_batch, y_batch, imgs_batch, epoch_i, step - 1) # Save reconstruction images if self.cfg.SAVE_IMAGE_STEP: if self.cfg.WITH_REC: if step % self.cfg.SAVE_IMAGE_STEP == 0: self._save_images(sess, self.train_image_path, x_batch, y_batch, imgs_batch, step - 1, epoch_i=epoch_i, silent=silent) # Save models if self.cfg.SAVE_MODEL_MODE == 'per_batch': if step % self.cfg.SAVE_MODEL_STEP == 0: self._save_model(sess, self.saver, step - 1, silent=silent) # Evaluate on full set if self.cfg.FULL_SET_EVAL_MODE == 'per_batch': if step % self.cfg.FULL_SET_EVAL_STEP == 0: self._eval_on_full_set(sess, epoch_i, step - 1, silent=silent) # Save model per epoch if self.cfg.SAVE_MODEL_MODE == 'per_epoch': if (epoch_i + 1) % self.cfg.SAVE_MODEL_STEP == 0: self._save_model(sess, self.saver, epoch_i) # Evaluate on valid set per epoch if self.cfg.FULL_SET_EVAL_MODE == 'per_epoch': if (epoch_i + 1) % self.cfg.FULL_SET_EVAL_STEP == 0: self._eval_on_full_set(sess, epoch_i, step - 1) # Evaluate on test set per epoch if self.cfg.TEST_SO_MODE == 'per_epoch': self._test(sess, during_training=True, epoch=epoch_i, step=step, mode='single') # Evaluate on multi-objects test set per epoch if self.cfg.TEST_MO_MODE == 'per_epoch': self._test(sess, during_training=True, epoch=epoch_i, step=step, mode='multi_obj') utils.thin_line() print('Epoch {}/{} done! Using time: {:.2f}'.format( epoch_i + 1, self.cfg.EPOCHS, time.time() - epoch_start_time)) utils.thick_line() print('Training finished! Using time: {:.2f}'.format(time.time() - self.start_time)) utils.thick_line() # Evaluate on test set after training if self.cfg.TEST_SO_MODE == 'after_training': self._test(sess, during_training=True, epoch='end', mode='single') # Evaluate on multi-objects test set after training if self.cfg.TEST_MO_MODE == 'after_training': self._test(sess, during_training=True, epoch='end', mode='multi_obj') utils.thick_line() print('All task finished! Total time: {:.2f}'.format(time.time() - self.start_time)) utils.thick_line()