class DataLoader(object): def __init__(self): self.view = View() self.train = DatasetSplit() self.val = DatasetSplit() def load_data(self, train_path, val_path, delimiter, shuffle_data=False): self.view.print_to_screen('Loading data...') self.train.load_data(train_path, delimiter) train_texts = self.train.get_texts() train_labels = self.train.get_labels() train_images = self.train.get_images() self.val.load_data(val_path, delimiter) val_texts = self.val.get_texts() val_labels = self.val.get_labels() val_images = self.val.get_images() self.view.print_to_screen('Train/Dev split: {:d}/{:d}'.format( len(train_texts), len(val_texts))) if shuffle_data: train_texts, train_labels, train_images = shuffle(train_texts, train_labels, train_images, random_state=10) self.set_training_data(train_texts, train_labels, train_images) self.set_val_data(val_texts, val_labels, val_images) def set_val_data(self, val_texts, val_labels, val_images): self.val.set_texts(val_texts) self.val.set_labels(val_labels) self.val.set_images(val_images) def set_training_data(self, train_texts, train_labels, train_images): self.train.set_texts(train_texts) self.train.set_labels(train_labels) self.train.set_images(train_images) def get_training_data(self): return self.train def get_val_data(self): return self.val
class EncodingExtractor(object): def __init__(self, train_dataset, val_dataset, root_dir, model_dir): self.train_dataset = train_dataset self.val_dataset = val_dataset self.root_dir = root_dir self.model_dir = model_dir self.view = View() def extract(self, extraction_parameters): checkpoint_dir = os.path.abspath(os.path.join(self.model_dir, 'checkpoints')) checkpoint_file = tf.train.latest_checkpoint(checkpoint_dir) batch_size = extraction_parameters.get_batch_size() image_resizer = ImageManipulator(extraction_parameters.get_output_image_width()) val_length = len(self.val_dataset.get_texts()) no_of_val_batches = (val_length // batch_size) + 1 graph = tf.Graph() with graph.as_default(): sess = tf.Session() with sess.as_default(): self.view.print_to_screen('Loading latest checkpoint: {}'.format(checkpoint_file)) saver = tf.train.import_meta_graph('{}.meta'.format(checkpoint_file)) saver.restore(sess, checkpoint_file) train_iterator, train_next_element = CustomIterator.create_iterator(self.train_dataset, batch_size) val_iterator, val_next_element = CustomIterator.create_iterator(self.val_dataset, batch_size) sess.run(train_iterator.initializer) sess.run(val_iterator.initializer) # Get the placeholders from the graph by name input_x = graph.get_operation_by_name('input_x').outputs[0] dropout_keep_prob = graph.get_operation_by_name('dropout_keep_prob').outputs[0] input_mask = graph.get_operation_by_name('input_mask').outputs[0] # Tensors we want to evaluate input_y = graph.get_operation_by_name('input_y').outputs[0] accuracy = graph.get_operation_by_name('accuracy/accuracy').outputs[0] sum = graph.get_operation_by_name('sum').outputs[0] input_tensor = ModelTensor(input_x, input_y, input_mask, dropout_keep_prob) correct = 0 for i in range(no_of_val_batches): val_batch = sess.run(val_next_element) path_list = [el.decode('UTF-8') for el in val_batch[2]] test_images_batch = image_resizer.preprocess_images(val_batch[2]) feed_dict = FeedDictCreator.create_feed_dict(input_tensor, val_batch, test_images_batch, 1) acc, img_sum = sess.run([accuracy, sum], feed_dict) correct += acc * batch_size thread = Thread(target=self.embedding_to_image, args=(self.root_dir, img_sum, path_list, extraction_parameters)) thread.start() self.compute_and_print_accuracy(correct, val_length, 'Test') train_length = len(self.train_dataset.get_texts()) no_of_train_batches = (train_length // batch_size) + 1 correct = 0 for i in range(no_of_train_batches): train_batch = sess.run(train_next_element) path_list = [el.decode('UTF-8') for el in train_batch[2]] train_images_batch = image_resizer.preprocess_images(train_batch[2]) feed_dict = FeedDictCreator.create_feed_dict(input_tensor, train_batch, train_images_batch, 1) acc, img_sum = sess.run([accuracy, sum], feed_dict) correct += acc * batch_size thread = Thread(target=self.embedding_to_image, args=(self.root_dir, img_sum, path_list, extraction_parameters)) thread.start() self.compute_and_print_accuracy(correct, train_length, 'Train') def embedding_to_image(self, root_dir, img_sum, test_img, extraction_parameters): x = extraction_parameters.get_separator_size() y = extraction_parameters.get_separator_size() encoding_height = extraction_parameters.get_encoding_height() superpixels_per_row = extraction_parameters.get_superpixel_per_row() superpixel_w = extraction_parameters.get_superpixel_w() output_image_width = extraction_parameters.get_output_image_width() superpixel_h = extraction_parameters.get_superpixel_h() for image, path in zip(img_sum, test_img): dir_names = path.split('/')[-3:] full_path = os.path.join(root_dir, os.path.join(dir_names[0], dir_names[1], dir_names[2].replace('.jpg', '.png'))) img = cv2.imread(path, cv2.IMREAD_COLOR) img = cv2.resize(img, (extraction_parameters.get_image_w(), extraction_parameters.get_image_h())) assert extraction_parameters.get_separator_size() + superpixels_per_row * superpixel_w \ <= extraction_parameters.get_image_w(), 'the image width is smaller than the visual word width' text_encoding_crop = image[0:encoding_height, 0:output_image_width, :] word_features = np.reshape(text_encoding_crop, (output_image_width * encoding_height * 3)) # C-like index ordering sp_i = 0 # superpixel index # write the embedding for row in list( range(y, int(y + extraction_parameters.get_superpixel_per_col() * superpixel_h), superpixel_h)): spw = 0 # counter for superpixels in row for col in list(range(x, int(x + superpixels_per_row * superpixel_w), superpixel_w)): ptl = sp_i * 3 ptr = (sp_i + 1) * 3 bgr = word_features[ptl:ptr] if len(bgr) == 0: break elif len(bgr) < 3: c = bgr.copy() c.resize(1, 3) bgr = c row_start = row row_end = row + superpixel_w for srow in range(row_start, row_end): for scol in range(col, col + superpixel_h): img[srow, scol] = bgr * 255 sp_i += 1 spw += 1 cv2.imwrite(full_path, img) def compute_and_print_accuracy(self, correct, split_dataset_length, phase): test_accuracy = correct / split_dataset_length self.view.print_to_screen( '{} accuracy: {} / {} = {}'.format(phase, int(correct), split_dataset_length, test_accuracy))
class ModelTrainer(object): def __init__(self, train_dataset, val_dataset): self.train_dataset = train_dataset self.val_dataset = val_dataset self.view = View() def train(self, training_params, model_params): patience = Patience(model_params.get_patience()) best_accuracy = Accuracy(0) output_width = training_params.get_output_image_width() image_resizer = ImageManipulator(output_width) with tf.Graph().as_default(): sess = tf.Session(config=tf.ConfigProto( allow_soft_placement=True, log_device_placement=False)) with sess.as_default(): cnn = TextImgCNN( sequence_length=self.train_dataset.get_texts().shape[1], num_classes=self.train_dataset.get_labels().shape[1], vocab_size=training_params.get_no_of_words_to_keep(), embedding_size=training_params.get_embedding_dim(), filter_sizes=list( map(int, training_params.get_filter_sizes().split(','))), num_filters=training_params.get_num_filters(), output_image_width=output_width, encoding_height=training_params.get_encoding_height(), l2_reg_lambda=0.0) train_iterator, next_train_batch = CustomIterator.create_iterator( self.train_dataset, training_params.get_batch_size()) test_iterator, next_test_element = CustomIterator.create_iterator( self.val_dataset, training_params.get_batch_size()) global_step = tf.Variable(0, name='global_step', trainable=False) optimizer = tf.train.AdamOptimizer(1e-3) grads_and_vars = optimizer.compute_gradients(cnn.loss) train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step) # Keep track of gradient values and sparsity (optional) grad_summaries = [] for g, v in grads_and_vars: if g is not None: grad_hist_summary = tf.summary.histogram( '{}/grad/hist'.format(v.name), g) sparsity_summary = tf.summary.scalar( '{}/grad/sparsity'.format(v.name), tf.nn.zero_fraction(g)) grad_summaries.append(grad_hist_summary) grad_summaries.append(sparsity_summary) grad_summaries_merged = tf.summary.merge(grad_summaries) out_dir = model_params.get_model_directory() self.view.print_to_screen('Writing to {}\n'.format(out_dir)) # Summaries for loss and test_accuracy loss_summary = tf.summary.scalar('loss', cnn.loss) acc_summary = tf.summary.scalar('test_accuracy', cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge( [loss_summary, acc_summary, grad_summaries_merged]) train_summary_dir = os.path.join(out_dir, 'summaries', 'train') train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Dev summaries dev_summary_op = tf.summary.merge([loss_summary, acc_summary]) saver = tf.train.Saver(tf.global_variables(), max_to_keep=1) file_logger = FileLogger(os.path.join(out_dir, 'result.txt')) file_logger.write_header(out_dir) sess.run(tf.global_variables_initializer()) train_length = len(self.train_dataset.get_texts()) no_of_training_batches = (train_length // training_params.get_batch_size()) + 1 input_tensor = ModelTensor(cnn.input_x, cnn.input_y, cnn.input_mask, cnn.dropout_keep_prob) for epoch in range(model_params.get_no_of_epochs()): sess.run(train_iterator.initializer) for i in range(no_of_training_batches): train_batch = sess.run(next_train_batch) train_images_batch = image_resizer.preprocess_images( train_batch[2]) feed_dict = FeedDictCreator.create_feed_dict( input_tensor, train_batch, train_images_batch, training_params.get_dropout_keep_probability()) _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) train_summary_writer.add_summary(summaries, step) current_step = tf.train.global_step(sess, global_step) training_result = TrainingResult(step, loss, accuracy) self.view.print_to_screen(str(training_result)) if current_step % model_params.evaluate_every == 0: self.view.print_to_screen('Evaluation:') val_length = len(self.val_dataset.get_texts()) no_of_val_batches = ( val_length // training_params.get_batch_size()) + 1 sess.run(test_iterator.initializer) correct = 0 for i in range(no_of_val_batches): test_batch = sess.run(next_test_element) test_images_batch = image_resizer.preprocess_images( test_batch[2]) feed_dict = FeedDictCreator.create_feed_dict( input_tensor, test_batch, test_images_batch, 1) step, summaries, loss, accuracy = sess.run([ global_step, dev_summary_op, cnn.loss, cnn.accuracy ], feed_dict) correct += accuracy * len(test_images_batch) test_accuracy = Accuracy(correct / val_length) partial_result = PartialResult( epoch, current_step, test_accuracy, best_accuracy, patience) self.view.print_to_screen(str(partial_result)) file_logger.write_partial_result_to_file( partial_result) if test_accuracy > best_accuracy: best_accuracy.set_value( test_accuracy.get_value()) patience.reset_patience() path = self.store_model( model_params, current_step, sess, saver) self.view.print_to_screen( 'Saved model checkpoint to {}\n'.format( path)) else: patience.decrement_patience() if patience.is_zero(): return @staticmethod def store_model(model_params, current_step, sess, saver): checkpoint_dir = os.path.abspath( os.path.join(model_params.get_model_directory(), 'checkpoints')) checkpoint_prefix = os.path.join(checkpoint_dir, 'model') return saver.save(sess, checkpoint_prefix, global_step=current_step)