def test_get_number_of_classes(self): exp_nr_of_classes_lbl = 0 with open(os.path.join(BOT_PROTOBUF_DIR, 'labels.txt')) as f: for ndx, ln in enumerate(f): pass exp_nr_of_classes_lbl = ndx + 1 exp_nr_of_classes_subf = 0 for ndx, dir in enumerate(os.listdir(BOT_TRAINING_DATA_DIR)): exp_nr_of_classes_subf = ndx exp_nr_of_classes_subf += 1 if not exp_nr_of_classes_lbl == exp_nr_of_classes_subf: print("Test invalid. Expected values are not matching %s != %s" % (exp_nr_of_classes_lbl, exp_nr_of_classes_subf)) return None txt_labels = utils.get_number_of_classes_by_labels(BOT_PROTOBUF_DIR) self.assertEqual(exp_nr_of_classes_lbl, txt_labels) self.assertEqual(exp_nr_of_classes_subf, txt_labels) sf_labels = utils.get_number_of_classes_by_subfolder( BOT_TRAINING_DATA_DIR) self.assertEqual(exp_nr_of_classes_subf, sf_labels) self.assertEqual(exp_nr_of_classes_lbl, sf_labels)
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1): """ Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name, restores the graph and runs it to the prediction enpoint with the image as input :param bot_id: bot_id, used to reference to correct model directory :param image_file: reference to the temporary image file to be classified :param network_name: name of the network type to be used :param return_labels: number of labels to return :return: the top n labels with probabilities, where n = return_labels """ # Get the model path model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id) # Get number of classes to predict protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir) # Get the preprocessing and network construction functions preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False) network_fn = network_factory.get_network_fn(network_name, number_of_classes) # Process the temporary image file into a Tensor of shape [widht, height, channels] image_tensor = tf.gfile.FastGFile(image_file, 'rb').read() image_tensor = tf.image.decode_image(image_tensor, channels=0) # Perform preprocessing and reshape into [network.default_width, network.default_height, channels] network_default_size = network_fn.default_image_size image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size) # Create an input batch of size one from the preprocessed image input_batch = tf.reshape(image_tensor, [1, 299, 299, 3]) # Create the network up to the Predictions Endpoint logits, endpoints = network_fn(input_batch) # Create a Saver() object to restore the network from the last checkpoint restorer = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() # Restore the variables of the network from the last checkpoint and run the graph restorer.restore(sess, tf.train.latest_checkpoint(model_path)) sess.run(endpoints) # Get the numpy array of predictions out of the predictions = endpoints['Predictions'].eval()[0] return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def test_bot_dataset(self): exp_train_set_size = utils.get_split_size(BOT_TRAINING_DATA_DIR, 'train') exp_val_set_size = utils.get_split_size(BOT_TRAINING_DATA_DIR, 'validation') exp_num_class = utils.get_number_of_classes_by_labels(BOT_PROTOBUF_DIR) train_set = data.get_split('train', BOT_PROTOBUF_DIR) validation_set = data.get_split('validation', BOT_PROTOBUF_DIR) self.assertTrue(train_set) self.assertTrue(type(train_set) is tf_slim.dataset.Dataset) self.assertEqual(train_set.num_classes, exp_num_class) self.assertEqual(train_set.num_samples, exp_train_set_size) self.assertTrue(validation_set) self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset) self.assertEqual(validation_set.num_classes, exp_num_class) self.assertEqual(validation_set.num_samples, exp_val_set_size)
def test_dataset_factory(self): train_set = factory.get_dataset('bot', 'train', BOT_PROTOBUF_DIR) validation_set = factory.get_dataset('bot', 'validation', BOT_PROTOBUF_DIR) self.assertTrue(train_set) self.assertTrue(type(train_set) is tf_slim.dataset.Dataset) self.assertEqual(train_set.num_classes, 5) self.assertEqual(train_set.num_samples, 3320) self.assertTrue(validation_set) self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset) self.assertEqual(validation_set.num_classes, 5) self.assertEqual(validation_set.num_samples, 350) bmw_models_bot_id = 'bmw_models' bmw_model_protobuf = dirs.get_protobuf_dir(bmw_models_bot_id) train_set = factory.get_dataset('bot', 'train', bmw_model_protobuf) validation_set = factory.get_dataset('bot', 'validation', bmw_model_protobuf) exp_num_classes = utils.get_number_of_classes_by_labels( bmw_model_protobuf) exp_train_set_size = utils.get_split_size(bmw_models_bot_id, 'train') exp_val_set_size = utils.get_split_size(bmw_models_bot_id, 'validation') self.assertTrue(train_set) self.assertTrue(type(train_set) is tf_slim.dataset.Dataset) self.assertEqual(train_set.num_classes, exp_num_classes) self.assertEqual(train_set.num_samples, exp_train_set_size) self.assertTrue(validation_set) self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset) self.assertEqual(validation_set.num_classes, exp_num_classes) self.assertEqual(validation_set.num_samples, exp_val_set_size)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None): """ :param split_name: :param dataset_dir: :param file_pattern: :param reader: :return: """ if split_name not in ['train', 'validation']: raise ValueError('illegal split name %s ' % split_name) num_classes = dataset_utils.get_number_of_classes_by_labels(dataset_dir) if not num_classes: raise FileNotFoundError('Dataset in %s not Found' % dataset_dir) if not file_pattern: file_pattern = _FILE_PATTERN file_pattern = os.path.join(dataset_dir, file_pattern % split_name) print("FILE PATTERN: %s" % file_pattern) # Allowing None in the signature so that dataset_factory can use the default. if reader is None: reader = tf.TFRecordReader keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='png'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) labels_to_names = None if dataset_utils.has_labels(dataset_dir): labels_to_names = dataset_utils.read_label_file(dataset_dir) bot_id = dirs.get_bot_id_from_dir(dataset_dir) training_data_dir = dirs.get_training_data_dir(bot_id) print("READING TRAINING DATA FROM: %s" % training_data_dir) if not bot_id: raise ValueError('bot id not recognized from dataset_dir %s' % dataset_dir) split_size = dataset_utils.get_split_size(training_data_dir, split_name, _SPLIT_FRAC) print("SPLIT SIZE: %s" % split_size) return slim.dataset.Dataset(data_sources=file_pattern, reader=reader, decoder=decoder, num_samples=split_size, items_to_descriptions=_ITEMS_TO_DESCRIPTIONS, num_classes=num_classes, labels_to_names=labels_to_names)