def train(setting_id, bot_id, hours=24, minutes=60, seconds=60, summary_secs=60): # root_model_dir = dirs.get_root_model_dir() bot_protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) bot_model_dir = dirs.get_transfer_model_dir(bot_id, setting_id, '_from_scratch') print(bot_model_dir) if _check_protobuf_dir(bot_protobuf_dir) and _check_bot_model_dir( bot_model_dir): print(""" STARTING TRANSFER LEARNING ROOT MODEL:\t%s TARGET MODEL:\t%s DATASET:\t%s """ % ('None', bot_model_dir, bot_protobuf_dir)) trainer.train( bot_model_dir=bot_model_dir, protobuf_dir=bot_protobuf_dir, model_name='inception_v4', max_train_time_sec=(seconds * minutes * hours), # seconds * minutes * hours * days optimization_params=None, log_every_n_steps=10, save_summaries_secs=summary_secs)
def train(setting_id, bot_id, hours=24, minutes=60, seconds=60, summary_secs=60): root_model_dir = dirs.get_root_model_dir() bot_protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) bot_model_dir = dirs.get_transfer_model_dir(bot_id, setting_id) _check_bot_model_dir(bot_model_dir) if _check_protobuf_dir(bot_protobuf_dir) and _check_root_model_dir( root_model_dir): print(""" STARTING TRANSFER LEARNING ROOT MODEL:\t%s TARGET MODEL:\t%s DATASET:\t%s """ % (root_model_dir, bot_model_dir, bot_protobuf_dir)) trainer.transfer_learning( root_model_dir=root_model_dir, bot_model_dir=bot_model_dir, protobuf_dir=bot_protobuf_dir, model_name='inception_v4', checkpoint_exclude_scopes=[ 'InceptionV4/Logits', 'InceptionV4/AuxLogits' ], trainable_scopes=['InceptionV4/Logits', 'InceptionV4/AuxLogits'], max_train_time_sec=(seconds * minutes * hours), # seconds * minutes * hours * days optimization_params=None, log_every_n_steps=10, save_summaries_secs=summary_secs)
def inference_on_image(bot_id, suffix, setting_id, image_file, network_name='inception_v4', return_labels=1): """ Loads the corresponding model checkpoint, network function and preprocessing routine based on bot_id and network_name, restores the graph and runs it to the prediction enpoint with the image as input :param bot_id: bot_id, used to reference to correct model directory :param image_file: reference to the temporary image file to be classified :param network_name: name of the network type to be used :param return_labels: number of labels to return :return: the top n labels with probabilities, where n = return_labels """ # Get the model path model_path = dirs.get_transfer_model_dir(bot_id+suffix, setting_id) # Get number of classes to predict protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) number_of_classes = dataset_utils.get_number_of_classes_by_labels(protobuf_dir) # Get the preprocessing and network construction functions preprocessing_fn = preprocessing_factory.get_preprocessing(network_name, is_training=False) network_fn = network_factory.get_network_fn(network_name, number_of_classes) # Process the temporary image file into a Tensor of shape [widht, height, channels] image_tensor = tf.gfile.FastGFile(image_file, 'rb').read() image_tensor = tf.image.decode_image(image_tensor, channels=0) # Perform preprocessing and reshape into [network.default_width, network.default_height, channels] network_default_size = network_fn.default_image_size image_tensor = preprocessing_fn(image_tensor, network_default_size, network_default_size) # Create an input batch of size one from the preprocessed image input_batch = tf.reshape(image_tensor, [1, 299, 299, 3]) # Create the network up to the Predictions Endpoint logits, endpoints = network_fn(input_batch) # Create a Saver() object to restore the network from the last checkpoint restorer = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() # Restore the variables of the network from the last checkpoint and run the graph restorer.restore(sess, tf.train.latest_checkpoint(model_path)) sess.run(endpoints) # Get the numpy array of predictions out of the predictions = endpoints['Predictions'].eval()[0] return map_predictions_to_labels(protobuf_dir, predictions, return_labels)
def _convert(bot_id, transfer_setting): training_data_dir = dirs.get_transfer_data_dir(bot_id, transfer_setting) protobuf_dir = dirs.get_transfer_proto_dir(bot_id, transfer_setting) print("READING FROM TRAINING DATA DIR: %s \n WRITING TO PROTOBUF DIR: %s" %(training_data_dir, protobuf_dir)) ''' readme_file = dirs.get_readme_file(transfer_setting) if not readme_file: print('No README found in %s' % training_data_dir) if readme_file: copyfile(readme_file, os.path.join(protobuf_dir, 'README')) ''' if _check_training_dir(training_data_dir) and _check_proto_dir(protobuf_dir): converter.run(training_data_dir, protobuf_dir, fract_validation=0.0)
from cnn_server.server import file_service as dirs d = dirs.get_transfer_proto_dir('cars', 9) print(d) print(dirs.get_bot_id_from_dir(d)) print(dirs.get_setting_id_from_dir(d))
def eval(bot_id, bot_suffix, setting_id=None, dataset_split='train', dataset_name='bot', model_name='inception_v4', preprocessing=None, moving_average_decay=None, tf_master=''): full_id = bot_id + bot_suffix if setting_id: protobuf_dir = dirs.get_transfer_proto_dir(bot_id, setting_id) else: protobuf_dir = dirs.get_protobuf_dir(bot_id) _check_dir(protobuf_dir) print("READIND FROM %s" % (protobuf_dir)) performance_data_dir = dirs.get_performance_data_dir(bot_id) # if os.listdir(performance_data_dir): # raise ValueError('%s is not empty' % performance_data_dir) tf.logging.set_verbosity(tf.logging.INFO) with tf.Graph().as_default(): tf_global_step = slim.get_or_create_global_step() ###################### # Select the dataset # ###################### dataset = dataset_factory.get_dataset(dataset_name, dataset_split, protobuf_dir) #################### # Select the model # #################### network_fn = nets_factory.get_network_fn( model_name, num_classes=(dataset.num_classes - LABELS_OFFSET), is_training=False) ############################################################## # Create a dataset provider that loads data from the dataset # ############################################################## provider = slim.dataset_data_provider.DatasetDataProvider( dataset, shuffle=False, common_queue_capacity=2 * BATCH_SIZE, common_queue_min=BATCH_SIZE) [image, label] = provider.get(['image', 'label']) label -= LABELS_OFFSET ##################################### # Select the preprocessing function # ##################################### preprocessing_name = preprocessing or model_name image_preprocessing_fn = preprocessing_factory.get_preprocessing( preprocessing_name, is_training=False) eval_image_size = EVAL_IMAGE_SIZE or network_fn.default_image_size image = image_preprocessing_fn(image, eval_image_size, eval_image_size) images, labels = tf.train.batch([image, label], batch_size=BATCH_SIZE, num_threads=NUM_THREADS, capacity=5 * BATCH_SIZE) #################### # Define the model # #################### logits, _ = network_fn(images) if moving_average_decay: variable_averages = tf.train.ExponentialMovingAverage( moving_average_decay, tf_global_step) variables_to_restore = variable_averages.variables_to_restore( slim.get_model_variables()) variables_to_restore[tf_global_step.op.name] = tf_global_step else: variables_to_restore = slim.get_variables_to_restore() predictions = tf.argmax(logits, 1) labels = tf.squeeze(labels) # Define the metrics: names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ 'Accuracy': slim.metrics.streaming_accuracy(predictions, labels), 'Recall_5': slim.metrics.streaming_recall_at_k(logits, labels, 5), }) # Print the summaries to screen. for name, value in names_to_values.items(): summary_name = 'eval/%s' % name op = tf.summary.scalar(summary_name, value, collections=[]) op = tf.Print(op, [value], summary_name) tf.add_to_collection(tf.GraphKeys.SUMMARIES, op) # TODO(sguada) use num_epochs=1 if MAX_NUM_BATCHES: num_batches = MAX_NUM_BATCHES else: # This ensures that we make a single pass over all of the data. num_batches = math.ceil(dataset.num_samples / float(BATCH_SIZE)) print(dataset.num_samples) print(dataset.num_classes)