def create_fine_tune_model(model_options, speech_network, vision_network):
"""Create siamese audio-visual similarity model for fine-tuning on unseen triplets."""
# TODO
# create clone of the vision network (so that it remains unchanged)
speech_network_clone = model_utils.create_and_copy_model(
speech_network, create_speech_network, model_options=model_options,
build_model=True) # TODO: figure out how to get this working without build (for MAML inner loop)
vision_network_clone = model_utils.create_and_copy_model(
vision_network, create_vision_network, model_options=model_options,
build_model=True) # TODO: figure out how to get this working without build (for MAML inner loop)
# freeze all model layers for transfer learning (except final dense layer)
freeze_index = -1
for layer in vision_network_clone.layers[:freeze_index]:
layer.trainable = False
for layer in speech_network_clone.layers[:freeze_index]:
layer.trainable = False
# use same objective as during training for fine-tuning
fine_tune_loss = get_training_objective(model_options)
few_shot_model = base.WeaklySupervisedModel(
speech_network_clone, vision_network_clone, fine_tune_loss,
mc_dropout=FLAGS.mc_dropout)
return few_shot_model
def create_fine_tune_model(model_options, vision_network, num_classes):
"""Create classification model for fine-tuning on unseen classes."""
# create clone of the vision network (so that it remains unchanged)
vision_network_clone = model_utils.create_and_copy_model(
vision_network, create_vision_network, model_options=model_options,
build_model=False)
# freeze model layers up to dense layer before relu & logits layer
if FLAGS.embed_layer == "dense":
freeze_index = -3 if model_options["dropout_rate"] is None else -4
# freeze all model layers for transfer learning (except final logits)
else:
freeze_index = -1
for layer in vision_network_clone.layers[:freeze_index]:
layer.trainable = False
# replace the logits layer with categorical logits layer for unseen classes
model_outputs = vision_network_clone.layers[-2].output
model_outputs = tf.keras.layers.Dense(num_classes)(model_outputs)
fine_tune_network = tf.keras.Model(
inputs=vision_network_clone.input, outputs=model_outputs)
fine_tune_loss = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
few_shot_model = base.BaseModel(
fine_tune_network, fine_tune_loss, mc_dropout=FLAGS.mc_dropout)
return few_shot_model
def create_fine_tune_model(model_options, vision_network, num_classes):
"""Create siamese similarity model for fine-tuning on unseen triplets."""
# create clone of the vision network (so that it remains unchanged)
vision_network_clone = model_utils.create_and_copy_model(
vision_network,
create_vision_network,
model_options=model_options,
build_model=True
) # TODO: figure out how to get this working without build (for MAML inner loop)
# freeze all model layers for transfer learning (except final dense layer)
freeze_index = -1
for layer in vision_network_clone.layers[:freeze_index]:
layer.trainable = False
# fine-tune image similarity network on siamese objective
if not FLAGS.classification:
fine_tune_network = vision_network_clone
# use same objective as during training for fine-tuning
fine_tune_loss = get_training_objective(model_options)
# add a categorical logits layer for fine-tuning on unseen classes
else:
vision_network_clone.layers[-1].trainable = False
model_outputs = vision_network_clone.output
model_outputs = tf.keras.layers.Dense(num_classes)(model_outputs)
fine_tune_network = tf.keras.Model(inputs=vision_network_clone.input,
outputs=model_outputs)
# use categorical cross entropy objective to fine-tune logits
fine_tune_loss = tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True)
few_shot_model = base.BaseModel(fine_tune_network,
fine_tune_loss,
mc_dropout=FLAGS.mc_dropout)
return few_shot_model