def train(setting_id,
bot_id,
hours=24,
minutes=60,
seconds=60,
summary_secs=60):
# root_model_dir = dirs.get_root_model_dir()
bot_protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
bot_model_dir = dirs.get_transfer_model_dir(bot_id, setting_id,
'_from_scratch')
print(bot_model_dir)
if _check_protobuf_dir(bot_protobuf_dir) and _check_bot_model_dir(
bot_model_dir):
print("""
STARTING TRANSFER LEARNING
ROOT MODEL:\t%s
TARGET MODEL:\t%s
DATASET:\t%s
""" % ('None', bot_model_dir, bot_protobuf_dir))
trainer.train(
bot_model_dir=bot_model_dir,
protobuf_dir=bot_protobuf_dir,
model_name='inception_v4',
max_train_time_sec=(seconds * minutes *
hours), # seconds * minutes * hours * days
optimization_params=None,
log_every_n_steps=10,
save_summaries_secs=summary_secs)
def train(setting_id,
bot_id,
hours=24,
minutes=60,
seconds=60,
summary_secs=60):
root_model_dir = dirs.get_root_model_dir()
bot_protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
bot_model_dir = dirs.get_transfer_model_dir(bot_id, setting_id)
_check_bot_model_dir(bot_model_dir)
if _check_protobuf_dir(bot_protobuf_dir) and _check_root_model_dir(
root_model_dir):
print("""
STARTING TRANSFER LEARNING
ROOT MODEL:\t%s
TARGET MODEL:\t%s
DATASET:\t%s
""" % (root_model_dir, bot_model_dir, bot_protobuf_dir))
trainer.transfer_learning(
root_model_dir=root_model_dir,
bot_model_dir=bot_model_dir,
protobuf_dir=bot_protobuf_dir,
model_name='inception_v4',
checkpoint_exclude_scopes=[
'InceptionV4/Logits', 'InceptionV4/AuxLogits'
],
trainable_scopes=['InceptionV4/Logits', 'InceptionV4/AuxLogits'],
max_train_time_sec=(seconds * minutes *
hours), # seconds * minutes * hours * days
optimization_params=None,
log_every_n_steps=10,
save_summaries_secs=summary_secs)
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1):
"""
Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name,
restores the graph and runs it to the prediction enpoint with the image as input
:param bot_id: bot_id, used to reference to correct model directory
:param image_file: reference to the temporary image file to be classified
:param network_name: name of the network type to be used
:param return_labels: number of labels to return
:return: the top n labels with probabilities, where n = return_labels
"""
# Get the model path
model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id)
# Get number of classes to predict
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir)
# Get the preprocessing and network construction functions
preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False)
network_fn = network_factory.get_network_fn(network_name, number_of_classes)
# Process the temporary image file into a Tensor of shape [widht, height, channels]
image_tensor = tf.gfile.FastGFile(image_file, 'rb').read()
image_tensor = tf.image.decode_image(image_tensor, channels=0)
# Perform preprocessing and reshape into [network.default_width, network.default_height, channels]
network_default_size = network_fn.default_image_size
image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size)
# Create an input batch of size one from the preprocessed image
input_batch = tf.reshape(image_tensor, [1, 299, 299, 3])
# Create the network up to the Predictions Endpoint
logits, endpoints = network_fn(input_batch)
# Create a Saver() object to restore the network from the last checkpoint
restorer = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
# Restore the variables of the network from the last checkpoint and run the graph
restorer.restore(sess, tf.train.latest_checkpoint(model_path))
sess.run(endpoints)
# Get the numpy array of predictions out of the
predictions = endpoints['Predictions'].eval()[0]
return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def _convert(bot_id, transfer_setting):
training_data_dir = dirs.get_transfer_data_dir(bot_id, transfer_setting)
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, transfer_setting)
print("READING FROM TRAINING DATA DIR: %s \n WRITING TO PROTOBUF DIR: %s" %(training_data_dir, protobuf_dir))
'''
readme_file = dirs.get_readme_file(transfer_setting)
if not readme_file:
print('No README found in %s' % training_data_dir)
if readme_file:
copyfile(readme_file, os.path.join(protobuf_dir, 'README'))
'''
if _check_training_dir(training_data_dir) and _check_proto_dir(protobuf_dir):
converter.run(training_data_dir, protobuf_dir, fract_validation=0.0)
from cnn_server.server import file_service as dirs
d = dirs.get_transfer_proto_dir('cars', 9)
print(d)
print(dirs.get_bot_id_from_dir(d))
print(dirs.get_setting_id_from_dir(d))
def eval(bot_id,
bot_suffix,
setting_id=None,
dataset_split='train',
dataset_name='bot',
model_name='inception_v4',
preprocessing=None,
moving_average_decay=None,
tf_master=''):
full_id = bot_id + bot_suffix
if setting_id:
protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id)
else:
protobuf_dir = dirs.get_protobuf_dir(bot_id)
_check_dir(protobuf_dir)
print("READIND FROM %s" % (protobuf_dir))
performance_data_dir = dirs.get_performance_data_dir(bot_id)
# if os.listdir(performance_data_dir):
# raise ValueError('%s is not empty' % performance_data_dir)
tf.logging.set_verbosity(tf.logging.INFO)
with tf.Graph().as_default():
tf_global_step = slim.get_or_create_global_step()
######################
# Select the dataset #
######################
dataset = dataset_factory.get_dataset(dataset_name, dataset_split,
protobuf_dir)
####################
# Select the model #
####################
network_fn = nets_factory.get_network_fn(
model_name,
num_classes=(dataset.num_classes - LABELS_OFFSET),
is_training=False)
##############################################################
# Create a dataset provider that loads data from the dataset #
##############################################################
provider = slim.dataset_data_provider.DatasetDataProvider(
dataset,
shuffle=False,
common_queue_capacity=2 * BATCH_SIZE,
common_queue_min=BATCH_SIZE)
[image, label] = provider.get(['image', 'label'])
label -= LABELS_OFFSET
#####################################
# Select the preprocessing function #
#####################################
preprocessing_name = preprocessing or model_name
image_preprocessing_fn = preprocessing_factory.get_preprocessing(
preprocessing_name, is_training=False)
eval_image_size = EVAL_IMAGE_SIZE or network_fn.default_image_size
image = image_preprocessing_fn(image, eval_image_size, eval_image_size)
images, labels = tf.train.batch([image, label],
batch_size=BATCH_SIZE,
num_threads=NUM_THREADS,
capacity=5 * BATCH_SIZE)
####################
# Define the model #
####################
logits, _ = network_fn(images)
if moving_average_decay:
variable_averages = tf.train.ExponentialMovingAverage(
moving_average_decay, tf_global_step)
variables_to_restore = variable_averages.variables_to_restore(
slim.get_model_variables())
variables_to_restore[tf_global_step.op.name] = tf_global_step
else:
variables_to_restore = slim.get_variables_to_restore()
predictions = tf.argmax(logits, 1)
labels = tf.squeeze(labels)
# Define the metrics:
names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({
'Accuracy':
slim.metrics.streaming_accuracy(predictions, labels),
'Recall_5':
slim.metrics.streaming_recall_at_k(logits, labels, 5),
})
# Print the summaries to screen.
for name, value in names_to_values.items():
summary_name = 'eval/%s' % name
op = tf.summary.scalar(summary_name, value, collections=[])
op = tf.Print(op, [value], summary_name)
tf.add_to_collection(tf.GraphKeys.SUMMARIES, op)
# TODO(sguada) use num_epochs=1
if MAX_NUM_BATCHES:
num_batches = MAX_NUM_BATCHES
else:
# This ensures that we make a single pass over all of the data.
num_batches = math.ceil(dataset.num_samples / float(BATCH_SIZE))
print(dataset.num_samples)
print(dataset.num_classes)