def baseline_finetune_bilinear(exp=1):
nb_epoch_finetune = 2
nb_epoch_after = 400
if exp == 1:
model = VGG16_bilinear(nb_class=nb_classes,
input_shape=input_shape,
freeze_conv=True)
model.name = 'baseline_vgg16_bilinear'
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
title = 'dtd_finetune_vgg16_bilinear'
elif exp == 2:
model = ResNet50_bilinear(nb_class=nb_classes,
input_shape=input_shape,
freeze_conv=True,
last_avg=False)
model.name = 'baseline_resnet50_bilinear'
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
# preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
title = 'dtd_finetune_resnet50_bilinear'
else:
raise NotImplementedError
# weight_path = '/home/kyu/.keras/models/dtd_finetune_vgg16_bilinear-baseline_vgg16_bilinear_1.weights_100'
weight_path = ''
dtd_finetune(model,
image_gen=image_gen,
title=title,
optimizer='sgd',
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
weight_path=weight_path,
load=False,
batch_size=32,
early_stop=False,
lr_decay=False,
lr=0.05)
def save_list_to_h5df():
import h5py
IMAGENET_PATH = '/home/kyu/.keras/datasets/ILSVRC2015'
TARGET_SIZE = (224, 224)
RESCALE_SMALL = 256
BATCH_SIZE = 16
NB_EPOCH = 70
VERBOSE = 1
SAVE_LOG = False
# ImageNet generator
imageNetLoader = ImageNetLoader(IMAGENET_PATH)
gen = ImageDataGeneratorAdvanced(TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
channelwise_std_normalization=True)
SAVE_PATH = IMAGENET_PATH + '/ImageSets/CLS-LOC/'
print("Saving the file to " + SAVE_PATH)
f = h5py.File(SAVE_PATH + 'imagenet_keras_pipeline_lists.h5', 'w')
f.create_dataset('train_list',
data=imageNetLoader.train_list,
compression='gzip',
compression_opts=9)
f.create_dataset('valid_list',
data=imageNetLoader.valid_list,
compression='gzip',
compression_opts=9)
f.create_dataset('test_list',
data=imageNetLoader.test_list,
compression='gzip',
compression_opts=9)
def run_model_with_config(model, config, title='mit_indoor',
image_gen=None,
verbose=(2, 2), nb_epoch_finetune=15, nb_epoch_after=50,
stiefel_observed=None, stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
if image_gen is None:
image_gen = ImageDataGeneratorAdvanced(TARGET_SIZE, RESCALE_SMALL, True,
horizontal_flip=True,
)
monitor_class = (O2Transform, SecondaryStatistic)
# monitor_metrics = ['weight_norm',]
# monitor_metrics = ['output_norm',]
monitor_metrics = ['matrix_image',]
finetune_model_with_config(model, mit_indoor_finetune, config, nb_classes, input_shape,
title=title, image_gen=image_gen, verbose=verbose,
nb_epoch_finetune=nb_epoch_finetune, nb_epoch_after=nb_epoch_after,
stiefel_observed=stiefel_observed, stiefel_lr=stiefel_lr)
def baseline_finetune_resnet(exp):
nb_epoch_finetune = 5
nb_epoch_after = 100
if exp == 1:
model = ResNet50_o1(denses=[],
nb_classes=nb_classes,
input_shape=input_shape,
freeze_conv=True)
model.name = 'baseline_resnet'
elif exp == 2:
last_conv_feature_maps = [1024, 512]
model = ResNet50_o1(denses=[],
nb_classes=nb_classes,
input_shape=input_shape,
freeze_conv=True,
last_conv_feature_maps=last_conv_feature_maps)
model.name = 'baseline_resnet_with_1x1'
else:
return
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
)
sun_finetune(model,
image_gen=image_gen,
title='sun_finetune_resnet',
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
batch_size=32,
early_stop=True)
def test_routine_resnet(config):
"""
Finetune the ResNet-DCov
Returns
-------
"""
nb_epoch_finetune = 2
nb_epoch_after = 50
title = 'test_sun_cov'
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
# preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
run_finetune(ResNet50_o2,
sun_finetune,
input_shape,
config,
image_gen,
nb_classes=nb_classes,
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
title=title,
verbose=(2, 1))
def baseline_finetune_bilinear(exp=1):
nb_epoch_finetune = 1
nb_epoch_after = 100
if exp == 1:
model = VGG16_bilinear(nb_class=nb_classes,
input_shape=input_shape,
freeze_conv=True)
model.name = 'baseline_vgg16_bilinear'
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
weight_path = '/home/kyu/.keras/models/mit_finetune_vgg16_bilinear-baseline_vgg16_bilinear_1.weights_100'
sun_finetune(model,
image_gen=image_gen,
title='mit_finetune_vgg16_bilinear',
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
weight_path=weight_path,
load=True,
batch_size=32,
early_stop=True,
lr_decay=False,
lr=0.05)
def run_routine_resnet(config,
verbose=(2, 2),
nb_epoch_finetune=15,
nb_epoch_after=50,
stiefel_observed=None,
stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
)
monitor_class = (O2Transform, SecondaryStatistic)
# monitor_metrics = ['weight_norm',]
# monitor_metrics = ['output_norm',]
monitor_metrics = [
'matrix_image',
]
run_model_with_config(ResNet50_o2,
config,
title='sun_resnet',
verbose=verbose,
image_gen=image_gen,
nb_epoch_after=nb_epoch_after,
nb_epoch_finetune=nb_epoch_finetune,
stiefel_lr=stiefel_lr,
stiefel_observed=stiefel_observed)
def run_routine_resnet_multibranch(config,
verbose=(2, 2),
nb_epoch_finetune=15,
nb_epoch_after=50,
stiefel_observed=None,
stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
)
run_model_with_config(ResNet50_o2_multibranch,
config,
title='sun_resnet_MB',
verbose=verbose,
image_gen=image_gen,
nb_epoch_after=nb_epoch_after,
nb_epoch_finetune=nb_epoch_finetune,
stiefel_lr=stiefel_lr,
stiefel_observed=stiefel_observed)
def test_routine_resnet(config,
verbose=(1, 2),
nb_epoch_finetune=1,
nb_epoch_after=50):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
# channelwise_std_normalization=True
)
run_finetune(ResNet50_o2,
dtd_finetune,
nb_classes=nb_classes,
input_shape=input_shape,
config=config,
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
image_gen=image_gen,
title='test_dtd_resnet',
verbose=verbose)
def baseline_finetune_resnet(exp):
nb_epoch_finetune = 10
nb_epoch_after = 50
if exp == 1:
model = ResNet50_o1(denses=[],
nb_classes=nb_classes,
input_shape=input_shape,
freeze_conv=True)
model.name = 'baseline_resnet'
elif exp == 2:
last_conv_feature_maps = [1024, 512]
model = ResNet50_o1(denses=[],
nb_classes=nb_classes,
input_shape=input_shape,
freeze_conv=True,
last_conv_feature_maps=last_conv_feature_maps)
else:
return
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
dtd_finetune(model,
image_gen=image_gen,
title='dtd_finetune_vgg16',
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
batch_size=32,
early_stop=True)
def run_routine_vgg(config, verbose=(2,2), nb_epoch_finetune=15, nb_epoch_after=50,
stiefel_observed=None, stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(TARGET_SIZE, RESCALE_SMALL, True,
horizontal_flip=True,
width_shift_range=0.2,
zoom_range=[0.7,1.3],
height_shift_range=0.2,
# preprocessing_function=preprocess_image_for_imagenet
)
monitor_class = (O2Transform, SecondaryStatistic)
# monitor_metrics = ['weight_norm',]
# monitor_metrics = ['output_norm',]
monitor_metrics = ['matrix_image',]
run_model_with_config(VGG16_o2, config, title='mit_indoor_VGG16',
verbose=verbose, image_gen=image_gen,
nb_epoch_after=nb_epoch_after, nb_epoch_finetune=nb_epoch_finetune,
stiefel_lr=stiefel_lr, stiefel_observed=stiefel_observed)
def test_routine_vgg(exp=1):
"""
Finetune the ResNet-DCov
Returns
-------
"""
nb_epoch_finetune = 10
nb_epoch_after = 10
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
config = get_experiment_settings(exp)
cov_branch = config.cov_branch
cov_mode = config.cov_mode
cov_regularizer = config.cov_regularizer
early_stop = config.early_stop
print("Running experiment {}".format(exp))
for param in config.params:
for mode in config.mode_list:
for cov_output in config.cov_outputs:
print("Run routine 1 param {}, mode {}, covariance output {}".
format(param, mode, cov_output))
model = VGG16_o2(
parametrics=param,
mode=mode,
cov_branch=cov_branch,
cov_mode=cov_mode,
nb_classes=nb_classes,
cov_branch_output=cov_output,
input_shape=input_shape,
freeze_conv=True,
cov_regularizer=cov_regularizer,
last_conv_feature_maps=config.last_conv_feature_maps)
sun_finetune(model,
title='sun_cov_{}_wv{}_{}'.format(
cov_branch, str(cov_output), cov_mode),
nb_epoch_after=nb_epoch_after,
nb_epoch_finetune=nb_epoch_finetune,
image_gen=image_gen,
batch_size=32,
early_stop=early_stop,
log=False,
verbose=1)
def run_routine_resnet(config,
verbose=(2, 2),
nb_epoch_finetune=10,
nb_epoch_after=50,
stiefel_observed=None,
stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
)
monitor_class = (O2Transform, SecondaryStatistic)
# monitor_metrics = ['weight_norm',]
# monitor_metrics = ['output_norm',]
monitor_metrics = [
'matrix_image',
]
if stiefel_observed is None:
run_finetune(ResNet50_o2,
dtd_finetune,
nb_classes=nb_classes,
input_shape=input_shape,
config=config,
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
image_gen=image_gen,
title='dtd_resnet50',
verbose=verbose,
monitor_classes=monitor_class,
monitor_measures=monitor_metrics)
else:
run_finetune_with_Stiefel_layer(ResNet50_o2,
dtd_finetune,
nb_classes=nb_classes,
input_shape=input_shape,
config=config,
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
image_gen=image_gen,
title='dtd_resnet50_stiefel',
verbose=verbose,
monitor_classes=monitor_class,
monitor_measures=monitor_metrics,
observed_keywords=stiefel_observed,
lr=stiefel_lr)
def test_loader():
loader = MITLoader(dirpath='/home/kyu/cvkyu/dataset/mit_indoor')
image_gen = ImageDataGeneratorAdvanced(TARGET_SIZE, RESCALE_SMALL, True,
horizontal_flip=True,
width_shift_range=0.1,
height_shift_range=0.1,
preprocessing_function=preprocess_image_for_imagenet
)
train = loader.generator('train', batch_size=16,
image_data_generator=image_gen,
save_to_dir='/home/kyu/cvkyu/plots',
save_prefix='mit_indoor', save_format='png'
)
train.next()
def baseline_finetune_vgg(exp=1):
nb_epoch_finetune = 10
nb_epoch_after = 50
if exp == 1:
model = VGG16_o1(denses=[256, 256], nb_classes=nb_classes, input_shape=input_shape, freeze_conv=True,
last_conv=False,
)
model.name = 'baseline_vgg16'
elif exp == 2:
model = VGG16_o1(denses=[256, 256], nb_classes=nb_classes, input_shape=input_shape, freeze_conv=True,
last_conv=True)
model.name = 'baseline_vgg16_with_1x1'
image_gen = ImageDataGeneratorAdvanced(TARGET_SIZE, RESCALE_SMALL,
random_crop=True,
horizontal_flip=True,
# width_shift_range=0.1,
# height_shift_range=0.1,
# preprocessing_function=preprocess_image_for_imagenet
)
mit_indoor_finetune(model, image_gen=image_gen, title='mit_indoor_finetune_vgg16',
nb_epoch_finetune=nb_epoch_finetune, nb_epoch_after=nb_epoch_after,
batch_size=32, early_stop=True)
def run_routine_vgg(config,
verbose=(2, 2),
nb_epoch_finetune=10,
nb_epoch_after=50,
stiefel_observed=None,
stiefel_lr=0.01):
"""
Finetune the ResNet-DCov
Returns
-------
"""
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
zoom_range=[0.5, 1.5],
width_shift_range=0.1,
height_shift_range=0.1,
preprocessing_function=preprocess_image_for_imagenet)
finetune_model_with_config(
VGG16_o2,
dtd_finetune,
config,
nb_classes,
input_shape,
title=config.title,
image_gen=image_gen,
verbose=verbose,
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
stiefel_observed=stiefel_observed,
stiefel_lr=stiefel_lr,
)
def baseline_finetune_vgg():
nb_epoch_finetune = 10
nb_epoch_after = 50
model = VGG16_o1(denses=[4096, 4096],
nb_classes=nb_classes,
input_shape=input_shape,
freeze_conv=True)
model.name = 'baseline_vgg16'
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
dtd_finetune(model,
image_gen=image_gen,
title='dtd_finetune_vgg16',
nb_epoch_finetune=nb_epoch_finetune,
nb_epoch_after=nb_epoch_after,
batch_size=32,
early_stop=True)
def run_residual_cov_resnet(exp):
"""
Finetune the ResNet-DCov
Returns
-------
m
"""
nb_epoch_finetune = 50
nb_epoch_after = 50
config = get_residual_cov_experiment(exp)
title = 'mit_indoor_residual_cov'
image_gen = ImageDataGeneratorAdvanced(TARGET_SIZE, RESCALE_SMALL, True,
horizontal_flip=True,
# preprocessing_function=preprocess_image_for_imagenet
# channelwise_std_normalization=True
)
run_finetune(ResNet50_o2, mit_indoor_finetune, input_shape, config, image_gen,
nb_classes=nb_classes,
nb_epoch_finetune=nb_epoch_finetune, nb_epoch_after=nb_epoch_after,
title=title, verbose=(2, 1))
def run_residual_cov_resnet(exp):
"""
Finetune the ResNet-DCov
Returns
-------
m
"""
nb_epoch_finetune = 40
nb_epoch_after = 50
config = get_residual_cov_experiment(exp)
cov_branch = config.cov_branch
cov_mode = config.cov_mode
cov_regularizer = config.cov_regularizer
early_stop = config.early_stop
image_gen = ImageDataGeneratorAdvanced(
TARGET_SIZE,
RESCALE_SMALL,
True,
horizontal_flip=True,
# channelwise_std_normalization=True
)
print("Running experiment {}".format(exp))
for param in config.params:
for mode in config.mode_list:
for cov_output in config.cov_outputs:
sess = K.get_session()
with sess.as_default():
title = 'dtd_residual_cov_{}_wv{}_{}'.format(
cov_branch, str(cov_output), cov_mode)
model = ResNet50_o2(
parametrics=param,
mode=mode,
cov_branch=cov_branch,
cov_mode=cov_mode,
nb_classes=nb_classes,
cov_branch_output=cov_output,
input_shape=input_shape,
last_avg=False,
freeze_conv=True,
cov_regularizer=cov_regularizer,
last_conv_feature_maps=config.last_conv_feature_maps,
epsilon=config.epsilon)
dtd_finetune(model,
title='finetune_' + title,
nb_epoch_after=0,
nb_epoch_finetune=nb_epoch_finetune,
log=True,
batch_size=config.batch_size,
early_stop=early_stop,
verbose=2,
image_gen=image_gen)
model.save_weights(get_tmp_weights_path(title +
model.name))
K.clear_session()
sess2 = K.get_session()
with sess2.as_default():
model = ResNet50_o2(
parametrics=param,
mode=mode,
cov_branch=cov_branch,
cov_mode=cov_mode,
nb_classes=nb_classes,
cov_branch_output=cov_output,
input_shape=input_shape,
last_avg=False,
freeze_conv=False,
cov_regularizer=cov_regularizer,
last_conv_feature_maps=config.last_conv_feature_maps,
epsilon=config.epsilon)
model.load_weights(get_tmp_weights_path(title +
model.name))
dtd_finetune(model,
title='retrain_' + title,
nb_epoch_after=0,
nb_epoch_finetune=nb_epoch_after,
log=True,
batch_size=config.batch_size / 4,
early_stop=early_stop,
verbose=2,
image_gen=image_gen)