def test_get_number_of_classes(self):
exp_nr_of_classes_lbl = 0
with open(os.path.join(BOT_PROTOBUF_DIR, 'labels.txt')) as f:
for ndx, ln in enumerate(f):
pass
exp_nr_of_classes_lbl = ndx + 1
exp_nr_of_classes_subf = 0
for ndx, dir in enumerate(os.listdir(BOT_TRAINING_DATA_DIR)):
exp_nr_of_classes_subf = ndx
exp_nr_of_classes_subf += 1
if not exp_nr_of_classes_lbl == exp_nr_of_classes_subf:
print("Test invalid. Expected values are not matching %s != %s" %
(exp_nr_of_classes_lbl, exp_nr_of_classes_subf))
return None
txt_labels = utils.get_number_of_classes_by_labels(BOT_PROTOBUF_DIR)
self.assertEqual(exp_nr_of_classes_lbl, txt_labels)
self.assertEqual(exp_nr_of_classes_subf, txt_labels)
sf_labels = utils.get_number_of_classes_by_subfolder(
BOT_TRAINING_DATA_DIR)
self.assertEqual(exp_nr_of_classes_subf, sf_labels)
self.assertEqual(exp_nr_of_classes_lbl, sf_labels)
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1):
"""
Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name,
restores the graph and runs it to the prediction enpoint with the image as input
:param bot_id: bot_id, used to reference to correct model directory
:param image_file: reference to the temporary image file to be classified
:param network_name: name of the network type to be used
:param return_labels: number of labels to return
:return: the top n labels with probabilities, where n = return_labels
"""
# Get the model path
model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id)
# Get number of classes to predict
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir)
# Get the preprocessing and network construction functions
preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False)
network_fn = network_factory.get_network_fn(network_name, number_of_classes)
# Process the temporary image file into a Tensor of shape [widht, height, channels]
image_tensor = tf.gfile.FastGFile(image_file, 'rb').read()
image_tensor = tf.image.decode_image(image_tensor, channels=0)
# Perform preprocessing and reshape into [network.default_width, network.default_height, channels]
network_default_size = network_fn.default_image_size
image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size)
# Create an input batch of size one from the preprocessed image
input_batch = tf.reshape(image_tensor, [1, 299, 299, 3])
# Create the network up to the Predictions Endpoint
logits, endpoints = network_fn(input_batch)
# Create a Saver() object to restore the network from the last checkpoint
restorer = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
# Restore the variables of the network from the last checkpoint and run the graph
restorer.restore(sess, tf.train.latest_checkpoint(model_path))
sess.run(endpoints)
# Get the numpy array of predictions out of the
predictions = endpoints['Predictions'].eval()[0]
return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def test_bot_dataset(self):
exp_train_set_size = utils.get_split_size(BOT_TRAINING_DATA_DIR,
'train')
exp_val_set_size = utils.get_split_size(BOT_TRAINING_DATA_DIR,
'validation')
exp_num_class = utils.get_number_of_classes_by_labels(BOT_PROTOBUF_DIR)
train_set = data.get_split('train', BOT_PROTOBUF_DIR)
validation_set = data.get_split('validation', BOT_PROTOBUF_DIR)
self.assertTrue(train_set)
self.assertTrue(type(train_set) is tf_slim.dataset.Dataset)
self.assertEqual(train_set.num_classes, exp_num_class)
self.assertEqual(train_set.num_samples, exp_train_set_size)
self.assertTrue(validation_set)
self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset)
self.assertEqual(validation_set.num_classes, exp_num_class)
self.assertEqual(validation_set.num_samples, exp_val_set_size)
def test_dataset_factory(self):
train_set = factory.get_dataset('bot', 'train', BOT_PROTOBUF_DIR)
validation_set = factory.get_dataset('bot', 'validation',
BOT_PROTOBUF_DIR)
self.assertTrue(train_set)
self.assertTrue(type(train_set) is tf_slim.dataset.Dataset)
self.assertEqual(train_set.num_classes, 5)
self.assertEqual(train_set.num_samples, 3320)
self.assertTrue(validation_set)
self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset)
self.assertEqual(validation_set.num_classes, 5)
self.assertEqual(validation_set.num_samples, 350)
bmw_models_bot_id = 'bmw_models'
bmw_model_protobuf = dirs.get_protobuf_dir(bmw_models_bot_id)
train_set = factory.get_dataset('bot', 'train', bmw_model_protobuf)
validation_set = factory.get_dataset('bot', 'validation',
bmw_model_protobuf)
exp_num_classes = utils.get_number_of_classes_by_labels(
bmw_model_protobuf)
exp_train_set_size = utils.get_split_size(bmw_models_bot_id, 'train')
exp_val_set_size = utils.get_split_size(bmw_models_bot_id,
'validation')
self.assertTrue(train_set)
self.assertTrue(type(train_set) is tf_slim.dataset.Dataset)
self.assertEqual(train_set.num_classes, exp_num_classes)
self.assertEqual(train_set.num_samples, exp_train_set_size)
self.assertTrue(validation_set)
self.assertTrue(type(validation_set) is tf_slim.dataset.Dataset)
self.assertEqual(validation_set.num_classes, exp_num_classes)
self.assertEqual(validation_set.num_samples, exp_val_set_size)
def get_split(split_name, dataset_dir, file_pattern=None, reader=None):
"""
:param split_name:
:param dataset_dir:
:param file_pattern:
:param reader:
:return:
"""
if split_name not in ['train', 'validation']:
raise ValueError('illegal split name %s ' % split_name)
num_classes = dataset_utils.get_number_of_classes_by_labels(dataset_dir)
if not num_classes:
raise FileNotFoundError('Dataset in %s not Found' % dataset_dir)
if not file_pattern:
file_pattern = _FILE_PATTERN
file_pattern = os.path.join(dataset_dir, file_pattern % split_name)
print("FILE PATTERN: %s" % file_pattern)
# Allowing None in the signature so that dataset_factory can use the default.
if reader is None:
reader = tf.TFRecordReader
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='png'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
labels_to_names = None
if dataset_utils.has_labels(dataset_dir):
labels_to_names = dataset_utils.read_label_file(dataset_dir)
bot_id = dirs.get_bot_id_from_dir(dataset_dir)
training_data_dir = dirs.get_training_data_dir(bot_id)
print("READING TRAINING DATA FROM: %s" % training_data_dir)
if not bot_id:
raise ValueError('bot id not recognized from dataset_dir %s' %
dataset_dir)
split_size = dataset_utils.get_split_size(training_data_dir, split_name,
_SPLIT_FRAC)
print("SPLIT SIZE: %s" % split_size)
return slim.dataset.Dataset(data_sources=file_pattern,
reader=reader,
decoder=decoder,
num_samples=split_size,
items_to_descriptions=_ITEMS_TO_DESCRIPTIONS,
num_classes=num_classes,
labels_to_names=labels_to_names)