def create_uncertainty_model(learning_rate=1e-3, num_hidden_units=20, type = 'mobilenet_v2'):
mu_input = Input(shape=(num_classes,))
if type == 'mobilenet_v2':
base_model = mobilenet_v2.MobileNetV2(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
elif type == 'vgg16':
base_model = vgg16.VGG16(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
elif type == 'resnet50':
base_model = resnet50.ResNet50(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
elif type == 'vgg19':
base_model = vgg19.VGG19(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
elif type == 'inception_v3':
base_model = inception_v3.InceptionV3(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
else:
base_model = mobilenet_v2.MobileNetV2(include_top=False, weights='imagenet', input_tensor=None,
input_shape=(224, 224, 3), pooling='avg', classes=num_classes)
base_model.trainable = False
beta = base_model.output
beta = Dense(num_hidden_units, activation='relu')(beta)
beta = Dense(num_hidden_units, activation='relu')(beta)
beta = Dense(num_hidden_units, activation='relu')(beta)
# beta = Dense(num_hidden_units,activation='relu')(beta)
beta = Dense(1, activation='sigmoid')(beta)
output = concatenate([mu_input, beta])
model = Model(inputs=[mu_input, base_model.input], outputs=output)
model.compile(loss=dirichlet_aleatoric_cross_entropy,
optimizer=Adam(lr=learning_rate),
metrics=[max_beta, min_beta]
)
return model
def prepare_image_data(paths):
images_root = paths['images_root']
captions_path = paths['train_captions_path']
output_path = paths['image_features_path']
if os.path.isfile(output_path):
print('Image prep: Output file already exists, doing nothing.')
return
# 'avg_pool' is the final layer in InceptionV3 before 'predictions'. That is the
# data used by VETE.
# NOTE(laser): This will download InceptionV3 and depends on pillow and h5py
base_model = inception_v3.InceptionV3(weights='imagenet', include_top=True)
model = Model(inputs=base_model.input,
outputs=base_model.get_layer('avg_pool').output)
with open(captions_path) as f:
image_metadata = json.load(f)
path = os.path.join(images_root, image_metadata['images'][0]['file_name'])
chunk_size = 512
image_ids = []
result_list = []
# OPTIMIZE(laser): In theory, image loading here is super slow. We're doing at
# least 2-3 times the number of copies we need. In practice, this only needs to
# run once over night.
chunk_idx = 0
for start in range(0, len(image_metadata['images']), chunk_size):
image_count = 0
image_list = []
for image_entry in image_metadata['images'][start:start + chunk_size]:
path = os.path.join(images_root, image_entry['file_name'])
# NOTE(laser): Paper mentions rescaling to 300x300 but default arguments
# in InceptionV3 docs say 299x299. Using that instead.
img = image.load_img(path, target_size=(299, 299))
x = image.img_to_array(img)
image_ids.append(image_entry['id'])
image_list.append(x)
image_count += 1
if image_count == chunk_size:
chunk_idx += 1
print('Loaded %s images (chunk %d)' %
(chunk_size * chunk_idx, chunk_idx - 1))
break
data = concat_np_list(image_list)
data = inception_v3.preprocess_input(data)
result = model.predict(data)
result_list.append(result)
print('Processed %s images (chunk %d)' %
(chunk_size * chunk_idx, chunk_idx - 1))
final_result = np.concatenate(result_list)
final_result = np.insert(final_result, 0, np.array(image_ids), axis=1)
final_result = np.sort(final_result, axis=0)
np.save(output_path, final_result)
def save_bottleneck_features():
# build the Inception V3 network
model = inception_v3.InceptionV3(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling='avg')
# Save the bottleneck features for the training data set
datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input)
generator = datagen.flow_from_directory(train_data_dir,
target_size=(img_width,
img_height),
batch_size=batch_size,
class_mode='sparse',
shuffle=False)
features = model.predict_generator(generator,
nb_train_samples // batch_size)
labels = np.eye(generator.num_classes, dtype='uint8')[generator.classes]
labels = labels[0:(nb_train_samples // batch_size) * batch_size]
np.save(open(output_dir + 'bottleneck_features_train.npy', 'wb'), features)
np.save(open(output_dir + 'bottleneck_labels_train.npy', 'wb'), labels)
# Save the bottleneck features for the validation data set
generator = datagen.flow_from_directory(validation_data_dir,
target_size=(img_width,
img_height),
batch_size=batch_size,
class_mode=None,
shuffle=False)
features = model.predict_generator(generator,
nb_validation_samples // batch_size)
labels = np.eye(generator.num_classes, dtype='uint8')[generator.classes]
labels = labels[0:(nb_validation_samples // batch_size) * batch_size]
np.save(open(output_dir + 'bottleneck_features_validation.npy', 'wb'),
features)
np.save(open(output_dir + 'bottleneck_labels_validation.npy', 'wb'),
labels)
def iterative_prune_model():
# build the inception v3 network
base_model = inception_v3.InceptionV3(include_top=False,
weights='imagenet',
pooling='avg',
input_shape=(299, 299, 3))
print('Model loaded.')
top_output = Dense(5, activation='softmax')(base_model.output)
# add the model on top of the convolutional base
model = Model(base_model.inputs, top_output)
del base_model
model.load_weights(tuned_weights_path)
# compile the model with a SGD/momentum optimizer
# and a very slow learning rate.
model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
# Set up data generators
train_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
train_steps = train_generator.n // train_generator.batch_size
test_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input)
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_height, img_width),
batch_size=val_batch_size,
class_mode='categorical')
val_steps = validation_generator.n // validation_generator.batch_size
# Evaluate the model performance before pruning
loss = model.evaluate_generator(validation_generator,
validation_generator.n //
validation_generator.batch_size)
print('original model validation loss: ', loss[0], ', acc: ', loss[1])
total_channels = get_total_channels(model)
n_channels_delete = int(math.floor(percent_pruning / 100 * total_channels))
# Incrementally prune the network, retraining it each time
percent_pruned = 0
# If percent_pruned > 0, continue pruning from previous checkpoint
if percent_pruned > 0:
checkpoint_name = ('inception_flowers_pruning_' + str(percent_pruned)
+ 'percent')
model = load_model(output_dir + checkpoint_name + '.h5')
while percent_pruned <= total_percent_pruning:
# Prune the model
apoz_df = get_model_apoz(model, validation_generator)
percent_pruned += percent_pruning
print('pruning up to ', str(percent_pruned),
'% of the original model weights')
model = prune_model(model, apoz_df, n_channels_delete)
# Clean up tensorflow session after pruning and re-load model
checkpoint_name = ('inception_flowers_pruning_' + str(percent_pruned)
+ 'percent')
model.save(output_dir + checkpoint_name + '.h5')
del model
tensorflow.python.keras.backend.clear_session()
tf.reset_default_graph()
model = load_model(output_dir + checkpoint_name + '.h5')
# Re-train the model
model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
checkpoint_name = ('inception_flowers_pruning_' + str(percent_pruned)
+ 'percent')
csv_logger = CSVLogger(output_dir + checkpoint_name + '.csv')
model.fit_generator(train_generator,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=validation_generator,
validation_steps=val_steps,
workers=4,
callbacks=[csv_logger])
# Evaluate the final model performance
loss = model.evaluate_generator(validation_generator,
validation_generator.n //
validation_generator.batch_size)
print('pruned model loss: ', loss[0], ', acc: ', loss[1])
def get_inceptionv3_model(weights='imagenet', input_tensor=None):
return inception_v3.InceptionV3(weights=weights, include_top=False, input_tensor=input_tensor)
def model_setting():
if not os.path.isdir('.\\h5'):
os.makedirs('.\\h5')
# whether load pre-trained model
if INITIAL_EPOCH != 0:
model = tf.keras.models.load_model('.\\h5\\' + '%s_%d.h5' %
(PROJECT_NAME, INITIAL_EPOCH))
else:
if MODEL_TYPE == 'DashNet':
from model.dashnet import DashNet
model = DashNet(input_shape=(MODEL_HEIGHT, MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
classes=CLASSES_NUM)
elif MODEL_TYPE == 'ShuffleNet':
from model.shufflenet import ShuffleNet
model = ShuffleNet(input_shape=(MODEL_HEIGHT, MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
classes=CLASSES_NUM,
groups=1)
elif MODEL_TYPE == 'SqueezeNet':
from model.squeezenet import SqueezeNet
model = SqueezeNet(
input_shape=(MODEL_HEIGHT, MODEL_WIDTH, MODEL_CHANNEL),
pooling='avg',
classes=CLASSES_NUM,
weights=None,
)
elif MODEL_TYPE == 'Xception':
model = xception.Xception(input_shape=(MODEL_HEIGHT, MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
include_top=True,
weights=None,
input_tensor=None,
classes=CLASSES_NUM)
elif MODEL_TYPE == 'NASNet':
model = nasnet.NASNetMobile(input_shape=(MODEL_HEIGHT, MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
include_top=True,
weights=None,
input_tensor=None,
classes=CLASSES_NUM)
elif MODEL_TYPE == 'Mobilenet':
model = mobilenet_v2.MobileNetV2(input_shape=(MODEL_HEIGHT,
MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
include_top=True,
weights=None,
input_tensor=None,
classes=CLASSES_NUM,
alpha=1.0)
elif MODEL_TYPE == 'InceptionV3':
model = inception_v3.InceptionV3(input_shape=(MODEL_HEIGHT,
MODEL_WIDTH,
MODEL_CHANNEL),
pooling='avg',
include_top=True,
weights=None,
input_tensor=None,
classes=CLASSES_NUM)
else:
print(
'Load Model Error!!! Please make sure you enter the correct model name!!!'
)
return model
def main(mode='train', gpu='0'):
os.environ["CUDA_VISIBLE_DEVICES"]=gpu
data = data_utils.ISIC2018_data(4)
num_classes = 7
# using resnet50
#base_model = resnet50.ResNet50(weights='imagenet', include_top=False, pooling='avg', input_shape=(224,224,3))
# using resnet152
#base_model = resnet152.ResNet152(weights='imagenet', include_top=False, pooling='avg', input_shape=(224,224,3))
# using resnet152
#base_model = resnet.ResNet152(weights='imagenet', include_top=False, pooling='avg', input_shape=(224,224,3))
# using densenet
#base_model = densenet.DenseNet201(weights='imagenet', include_top=False, pooling='avg', input_shape=(224,224,3))
# inception v3
base_model = inception_v3.InceptionV3(weights='imagenet', include_top=False, pooling='avg', input_shape=(224,224,3))
model = models.Sequential()
model.add(base_model)
# remove fully connected according to paper
model.add(layers.Dense(2048, activation='relu'))
model.add(layers.Dense(1000, activation='relu'))
model.add(layers.Dense(1000, activation='relu'))
model.add(layers.Dense(num_classes, activation='softmax', name='fc7'))
base_model.trainable = False
#model.compile(loss=median_weight_class_loss,
#model.compile(loss='categorical_crossentropy',
model.compile(loss=focal_loss,
optimizer=SGD(lr=0.001, momentum=0.9, decay=0.0),
metrics=[metrics.categorical_accuracy])
model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/inception_v3_keras_pre/model_fc_3'
#model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/resnet152_keras_pre/model_fc'
#model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/resnet152_keras_pre/model_fc'
#model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/resnet50_keras_pre/model_cw'
#model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/resnet50_keras_pre/model_fc_cw'
#model_dir = '/home/jiaxin/myGithub/Reverse_CISI_Classification/src/resnet50_keras_pre/model_fc_cw_nor'
os.makedirs(model_dir, exist_ok=True)
print('model_dir', model_dir)
est = tf.keras.estimator.model_to_estimator(keras_model=model,
model_dir=model_dir)
train_spec = tf.estimator.TrainSpec(input_fn=lambda: data.read_record('train'))
eval_spec = tf.estimator.EvalSpec(input_fn=lambda: data.read_record('train_evaluation'))
if mode == 'train':
tf.estimator.train_and_evaluate(est, train_spec, eval_spec)
elif mode == 'evaluate':
with tf.Session() as sess:
try:
x, y = data.read_record('evaluate')
sess.run(tf.global_variables_initializer())
y_list = []
while True:
_, y_ = sess.run([x, y])
y_list.extend((np.argmax(y_, axis=1)))
except tf.errors.OutOfRangeError:
print('load finish labels')
# test
#cnt = 0
#while True:
# try:
# x, y = data.read_record('evaluate')
# sess.run(tf.global_variables_initializer())
# y_list_ = []
# while True:
# _, y_ = sess.run([x, y])
# y_list_.extend((np.argmax(y_, axis=1)))
# except tf.errors.OutOfRangeError:
# cnt += 1
# print(cnt)
# assert all([a==b for a, b in zip(y_list, y_list_)])
pp = []
while True:
predictions = est.predict(input_fn=lambda: data.read_record('evaluate'))
predictions_list = []
for pre in predictions:
p = np.argmax(pre['fc7'])
predictions_list.append(p)
statistics_ = statistics.statistics(hps, mode='evaluate')
statistics_.add_labels_predictions(predictions_list, y_list)
statistics_.get_acc_normal()
result = statistics_.get_acc_imbalanced()
np.save('predictions_label_fc_3', [predictions_list, y_list])
#np.save('predictions_label_fc_without_fulcon', [predictions_list, y_list])
pp.append(result)
print('---')
np.save('result_fc_3', pp)
#np.save('result_fc_without_fulcon', pp)
time.sleep(120)
def tune_model():
# Build the Inception V3 network.
base_model = inception_v3.InceptionV3(include_top=False,
weights='imagenet',
pooling='avg')
print('Model loaded.')
# build a classifier model to put on top of the convolutional model
top_input = Input(shape=base_model.output_shape[1:])
top_output = Dense(5, activation='softmax')(top_input)
top_model = Model(top_input, top_output)
# Note that it is necessary to start with a fully-trained classifier,
# including the top classifier, in order to successfully do fine-tuning.
top_model.load_weights(top_model_weights_path)
# add the model on top of the convolutional base
model = Model(inputs=base_model.inputs,
outputs=top_model(base_model.outputs))
# Set all layers up to 'mixed8' to non-trainable (weights will not be updated)
last_train_layer = model.get_layer(name='mixed8')
for layer in model.layers[:model.layers.index(last_train_layer)]:
layer.trainable = False
# Compile the model with a SGD/momentum optimizer and a very slow learning rate.
model.compile(loss='categorical_crossentropy',
optimizer=SGD(lr=1e-4, momentum=0.9),
metrics=['accuracy'])
# Prepare data augmentation configuration
train_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(
preprocessing_function=inception_v3.preprocess_input)
train_generator = train_datagen.flow_from_directory(
train_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validation_data_dir,
target_size=(img_height, img_width),
batch_size=batch_size,
class_mode='categorical')
loss = model.evaluate_generator(validation_generator,
nb_validation_samples // batch_size)
print('Model validation performance before fine-tuning:', loss)
csv_logger = CSVLogger(output_dir + 'model_tuning.csv')
# fine-tune the model
model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=tune_epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
workers=4,
callbacks=[csv_logger])
model.save_weights(tuned_weights_path)
from tensorflow.python.keras.applications import vgg16
from tensorflow.python.keras.preprocessing import image
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.layers import Dense, GlobalAveragePooling2D
from tensorflow.python.keras import backend as K
from keras.applications.imagenet_utils import preprocess_input, decode_predictions
from IPython.display import Image
from keras.preprocessing import image
import numpy as np
print(tf.keras.__version__)
print(tf.__version__)
model = inception_v3.InceptionV3(weights='imagenet', include_top=True)
image_path = "./imagenet/"
img_path = os.path.join(image_path, 'cow.jpg')
img = image.load_img(img_path, target_size=(299, 299))
x = image.img_to_array(img)
x = np.expand_dims(x, axis=0)
x = inception_v3.preprocess_input(x)
print('Input image shape:', x.shape)
preds = model.predict(x)
print(' ')
print('Predicted:', decode_predictions(preds))
Image(img_path)