def create_fine_tune_model(model_options, speech_network, vision_network): """Create siamese audio-visual similarity model for fine-tuning on unseen triplets.""" # TODO # create clone of the vision network (so that it remains unchanged) speech_network_clone = model_utils.create_and_copy_model( speech_network, create_speech_network, model_options=model_options, build_model=True) # TODO: figure out how to get this working without build (for MAML inner loop) vision_network_clone = model_utils.create_and_copy_model( vision_network, create_vision_network, model_options=model_options, build_model=True) # TODO: figure out how to get this working without build (for MAML inner loop) # freeze all model layers for transfer learning (except final dense layer) freeze_index = -1 for layer in vision_network_clone.layers[:freeze_index]: layer.trainable = False for layer in speech_network_clone.layers[:freeze_index]: layer.trainable = False # use same objective as during training for fine-tuning fine_tune_loss = get_training_objective(model_options) few_shot_model = base.WeaklySupervisedModel( speech_network_clone, vision_network_clone, fine_tune_loss, mc_dropout=FLAGS.mc_dropout) return few_shot_model
def create_fine_tune_model(model_options, vision_network, num_classes): """Create classification model for fine-tuning on unseen classes.""" # create clone of the vision network (so that it remains unchanged) vision_network_clone = model_utils.create_and_copy_model( vision_network, create_vision_network, model_options=model_options, build_model=False) # freeze model layers up to dense layer before relu & logits layer if FLAGS.embed_layer == "dense": freeze_index = -3 if model_options["dropout_rate"] is None else -4 # freeze all model layers for transfer learning (except final logits) else: freeze_index = -1 for layer in vision_network_clone.layers[:freeze_index]: layer.trainable = False # replace the logits layer with categorical logits layer for unseen classes model_outputs = vision_network_clone.layers[-2].output model_outputs = tf.keras.layers.Dense(num_classes)(model_outputs) fine_tune_network = tf.keras.Model( inputs=vision_network_clone.input, outputs=model_outputs) fine_tune_loss = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) few_shot_model = base.BaseModel( fine_tune_network, fine_tune_loss, mc_dropout=FLAGS.mc_dropout) return few_shot_model
def create_fine_tune_model(model_options, vision_network, num_classes): """Create siamese similarity model for fine-tuning on unseen triplets.""" # create clone of the vision network (so that it remains unchanged) vision_network_clone = model_utils.create_and_copy_model( vision_network, create_vision_network, model_options=model_options, build_model=True ) # TODO: figure out how to get this working without build (for MAML inner loop) # freeze all model layers for transfer learning (except final dense layer) freeze_index = -1 for layer in vision_network_clone.layers[:freeze_index]: layer.trainable = False # fine-tune image similarity network on siamese objective if not FLAGS.classification: fine_tune_network = vision_network_clone # use same objective as during training for fine-tuning fine_tune_loss = get_training_objective(model_options) # add a categorical logits layer for fine-tuning on unseen classes else: vision_network_clone.layers[-1].trainable = False model_outputs = vision_network_clone.output model_outputs = tf.keras.layers.Dense(num_classes)(model_outputs) fine_tune_network = tf.keras.Model(inputs=vision_network_clone.input, outputs=model_outputs) # use categorical cross entropy objective to fine-tune logits fine_tune_loss = tf.keras.losses.SparseCategoricalCrossentropy( from_logits=True) few_shot_model = base.BaseModel(fine_tune_network, fine_tune_loss, mc_dropout=FLAGS.mc_dropout) return few_shot_model