class ImgNet(object):
def __init__(self, name="resnet"):
# 学習済みのVGG16をロード
# 構造とともに学習済みの重みも読み込まれる
if name == "mobilenet":
self.model = MobileNet(weights='imagenet')
elif name == "resnet":
self.model = ResNet50(weights='imagenet')
else:
self.model = ResNet50(weights='imagenet')
self.model.summary()
def array2predict(self, x):
# 3次元テンソル（rows, cols, channels) を
# 4次元テンソル (samples, rows, cols, channels) に変換
# 入力画像は1枚なのでsamples=1でよい
x = np.expand_dims(x, axis=0)
# Top-5のクラスを予測する
# VGG16の1000クラスはdecode_predictions()で文字列に変換される
preds = self.model.predict(preprocess_input(x))
results = decode_predictions(preds, top=5)[0]
return results
def file2predict(self, filename):
# 引数で指定した画像ファイルを読み込む
# サイズはVGG16のデフォルトである224x224にリサイズされる
img = image.load_img(filename, target_size=(224, 224))
# 読み込んだPIL形式の画像をarrayに変換
x = image.img_to_array(img)
results = self.array2predict(x)
return results
#!/usr/bin/python
# # -*- coding: UTF-8 -*-
from tensorflow.keras.preprocessing import image
import numpy as np
import cv2
from tensorflow.keras.models import load_model
from tensorflow.keras.applications.mobilenet import MobileNet
from tensorflow.keras.applications.mobilenet import preprocess_input, decode_predictions
#include_top=True，完整的模型
#include_top=False，去掉最后的3个全连接层，用来做fine-tuning专用，专门开源了这类模型。
model = MobileNet(weights='imagenet')
print(model.summary())
img_path = "elephant.jpg"
img = image.load_img(img_path, target_size=(224, 224))
#将输入数据转换为0~1之间
img = image.img_to_array(img) / 255.0
# 为batch添加第四维,axis=0表示在0位置添加,因为MobileNet的Iput层结构是（None,224,224,3）
img = np.expand_dims(img, axis=0)
print(img.shape)
predictions = model.predict(img)
print('Predicted:', decode_predictions(predictions, top=3)[0])
print(predictions)
description = decode_predictions(predictions, top=3)[0][0][1]
src = cv2.imread(img_path)
cv2.putText(src, description, (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.8,
(255, 0, 0), 2)
# Angkan Biswas # 26.04.2020 # To load pretrained model ''' For the first time model will be downloaded from Keras repository & it will be saved in users home directory. Next time model will be loaded from the users home directory, i.e., /home/dell/.keras/models/''' from tensorflow.keras.applications.vgg16 import VGG16 from tensorflow.keras.applications.mobilenet import MobileNet vgg16Model = VGG16() # Load pretrained model VGG16 [size = 528 MB] vgg16Model.summary() # Display model arcitracture del vgg16Model # Delete loaded model. It is nessary when multiple models need to loaded mobileModel =MobileNet() # Load pretrained mobile [size = 17 MB] mobileModel.summary() # Display model arcitracture
x = tf.keras.layers.Conv2D(num_classes, 1, name='conv_preds')(x)
x = tf.keras.layers.Activation('softmax', name='act_softmax')(x)
x = tf.keras.layers.Reshape((10,), name='reshape_2')(x)
model = tf.keras.Model(inputs=image_input, outputs=x)
# ImageNet weights, basic fully connected (dense) top
elif option == 3:
image_input = tf.keras.Input(shape=x_train.shape[1:], name='image_input')
x = MobileNet(weights='imagenet', include_top=False)(image_input)
x = tf.keras.layers.Flatten(name='flatten')(x)
x = tf.keras.layers.Dense(4096, activation='relu', name='fc1')(x)
x = tf.keras.layers.Dense(4096, activation='relu', name='fc2')(x)
x = tf.keras.layers.Dense(num_classes, activation='softmax', name='predictions')(x)
model = tf.keras.Model(inputs=image_input, outputs=x)
model.summary()
# Select loss, optimizer, metric
model.compile(loss='categorical_crossentropy',
optimizer=tf.train.AdamOptimizer(0.001),
metrics=['accuracy'])
# Train
start = timer()
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
# callbacks=[TensorBoardColabCallback(tbc)]
class Classifier:
def __init__(self):
self.categories = list('ABCDEFGHIJKLMNOPQRSTUVWXYZ')
self.category_map = {i: v for i, v in enumerate(self.categories)}
self.num_classes = len(self.categories)
self.img_size = 224
self.batch_size = 64
self.train_path, self.test_path = None, None
self.train_generator, self.test_generator = None, None
self.step_size_train, self.step_size_valid = None, None
self.images, self.labels = None, None
self.classifier = None
self.history = None
self.grad_cam_names = None
self.save_folder = None
self.feature_extractor = None
self.latent_vectors = None
self.latent_path = None
self.latent_test = None
self.latent_train = None
def generate_data(self, train_path, test_path, batch_size, figsize=(10,10), fontsize=16):
self.train_path = train_path
self.test_path = test_path
self.batch_size = batch_size
test_datagen = ImageDataGenerator(
rescale=1/255.)
train_datagen = ImageDataGenerator(
rescale=1/255.,
brightness_range=[.9, 1.],
rotation_range=5,
zoom_range=.1,
width_shift_range=.1,
height_shift_range=.1)
self.train_generator = train_datagen.flow_from_directory(
self.train_path,
shuffle=True,
target_size=(self.img_size, self.img_size),
color_mode='rgb',
batch_size=batch_size,
seed=0,
class_mode="categorical")
self.test_generator = test_datagen.flow_from_directory(
self.test_path,
shuffle=True,
target_size=(self.img_size, self.img_size),
color_mode='rgb',
batch_size=batch_size,
seed=0,
class_mode="categorical")
_, axes = plt.subplots(6, 6, figsize=figsize)
for i, category in enumerate(self.categories[:6]):
path = self.train_path + '/' + category
images = os.listdir(path)
for j in range(6):
image = cv2.imread(path + '/' + images[j])
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
axes[i, j].imshow(image)
axes[i, j].set(xticks=[], yticks=[])
axes[i, j].set_title(category, color = 'tomato').set_size(fontsize)
plt.suptitle('Vanilla Data').set_size(2*fontsize)
plt.tight_layout()
images, labels = self.train_generator.next()
_, axes = plt.subplots(6, 6, figsize=figsize)
for i in range(6):
for j in range(6):
image = images[i+j]
label = self.category_map[np.argmax(labels[i+j])]
axes[i, j].imshow(image)
axes[i, j].set(xticks=[], yticks=[])
axes[i, j].set_title(label, color = 'tomato').set_size(fontsize)
plt.suptitle('Augmented Data').set_size(2*fontsize)
plt.tight_layout()
self.step_size_train=int(self.train_generator.n // self.train_generator.batch_size)
self.step_size_valid=int(self.test_generator.n // self.test_generator.batch_size)
def notify(self, fig):
fig.savefig('tmp.jpg')
with open('tmp.jpg', 'rb') as f:
telegram_send.send(images=[f])
os.remove('tmp.jpg')
def plot_accuracy(self, history):
f, axes = plt.subplots(1, 2, figsize=(12, 4))
accuracy = history.history['accuracy']
loss = history.history['loss']
val_accuracy = history.history['val_accuracy']
val_loss = history.history['val_loss']
print('Training accuracy: {:.{}f}'.format(np.max(accuracy), 3))
print('Training loss: {:.{}f}'.format(np.max(loss), 3))
print('Validation accuracy: {:.{}f}'.format(np.max(val_accuracy), 3))
print('Validation loss: {:.{}f}'.format(np.max(val_loss), 3))
axes[0].plot(history.history['accuracy'])
axes[0].plot(history.history['val_accuracy'])
axes[0].set_title('Model accuracy')
axes[0].set(ylabel = 'accuracy', xlabel = 'Epoch')
axes[0].legend(['Train', 'Test'], loc='upper left')
axes[1].plot(history.history['loss'])
axes[1].plot(history.history['val_loss'])
axes[1].set_title('Model loss')
axes[1].set(ylabel = 'Loss', xlabel = 'Epoch')
axes[1].legend(['Train', 'Test'], loc='upper left')
return f
def set_feature_extractor(self, name = 'mobilenet', summary = False):
if name == 'mobilenet':
self.feature_extractor = MobileNet(input_shape = (self.img_size, self.img_size, 3), include_top=True,weights ='imagenet')
output = self.feature_extractor.layers[-6].output
self.feature_extractor = tf.keras.Model(self.feature_extractor.inputs, output)
if summary:
self.feature_extractor.summary()
def extract_and_save(self, latent_path, latent_vectors, save=True):
'''model: Model used to extract encoded features
generator: yields (x_batch, y_batch)
'''
self.latent_vectors = latent_vectors
self.latent_path = latent_path
if not save:
return
for folder in ['train', 'test']:
save_path = os.path.join(latent_path, folder)
if os.path.exists(save_path) and os.path.isdir(save_path):
shutil.rmtree(save_path)
os.makedirs(save_path, exist_ok=True)
template = 'batch_{}.h5'
batch = 0
for generator in [self.train_generator, self.test_generator]:
for x_batch, y_batch in tqdm.tqdm(generator):
IPython.display.clear_output(wait=True)
features = self.feature_extractor.predict(x_batch)
file_path = os.path.join(save_path, template.format(batch))
with h5py.File(file_path, 'w') as file:
# encoded features and hard labels
file.create_dataset('features', data=features)
file.create_dataset('labels', data=y_batch)
batch += 1
if folder == 'train':
if batch >= self.step_size_train:
break
else:
if batch >= self.step_size_valid:
break
def load_data(self, folder):
'''yields (x_batch, y_batch) for model.fit()
'''
root_path = os.path.join(self.latent_path, folder)
while True:
for file_path in os.listdir(root_path):
with h5py.File(os.path.join(root_path, file_path), 'r') as file:
yield (np.array(file['features']), np.array(file['labels']))
def clear_session(self):
tf.keras.backend.clear_session()
def train(self,
lr=None,
optimizer=None,
epochs=None,
decay_lr=False,
save_folder=None,
notification = False):
self.save_folder = save_folder
# shape of VGG16 encoded features
inputs = layers.Input(shape=self.latent_vectors)
x = layers.Dense(self.num_classes, activation='softmax')(inputs)
self.classifier = tf.keras.Model(inputs, x)
self.classifier.compile(optimizer=optimizer(lr=lr),
loss='categorical_crossentropy',
metrics=['accuracy'])
def lr_decay(epoch):
alpha, decay = 1, 1
return lr / (alpha + decay * epoch)
callback_learning_rate = tf.keras.callbacks.LearningRateScheduler(lr_decay, verbose=True)
plot_losses = PlotLosses()
callback_is_nan = tf.keras.callbacks.TerminateOnNaN()
callback_early = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta = .001, patience = 10)
callbacks = [plot_losses, callback_is_nan, callback_early]
callbacks += [callback_learning_rate] if decay_lr else []
self.latent_train, self.latent_test = 'train', 'test'
self.history = self.classifier.fit(
x = self.load_data(self.latent_train),
epochs=epochs,
workers=15,
steps_per_epoch=self.step_size_train,
validation_steps=self.step_size_valid,
validation_data=self.load_data(self.latent_test),
callbacks=callbacks)
fig = self.plot_accuracy(self.history)
if save_folder:
asl_detection.save.save(save_folder, 'acc_loss', fig=fig)
asl_detection.save.save(save_folder, 'model', self.classifier)
if notification:
self.notify(fig)
self.images, self.labels = self.test_generator.next()
def _visualize_feature_maps(self, image, _layers, scale):
model_layers = self.feature_extractor.layers
# Extracts the outputs
layer_outputs = [layer.output for layer in self.feature_extractor.layers]
# Creates a model that will return these outputs, given the model input
activation_model = tf.keras.Model(inputs=self.feature_extractor.inputs, outputs=layer_outputs)
# get activations
activations = activation_model.predict(image)
images_per_row = 4; count = -1
# Displays the feature maps
for layer, layer_activation in zip(model_layers, activations):
if not isinstance(layer, layers.Conv2D):
continue
count += 1
# show first 3 conv layers
if count != _layers:
continue
n_features = layer_activation.shape[-1] # Number of features in the feature map
size = layer_activation.shape[1] #The feature map has shape (1, size, size, n_features).
n_cols = n_features // images_per_row # Tiles the activation channels in this matrix
display_grid = np.zeros((size * n_cols, images_per_row * size))
for col in range(n_cols): # Tiles each filter into a big horizontal grid
for row in range(images_per_row):
channel_image = layer_activation[0, :, :, col * images_per_row + row]
# Post-processes the feature to make it visually palatable
channel_image -= channel_image.mean()
channel_image /= channel_image.std() + 1e-8
channel_image *= 64
channel_image += 128
channel_image = np.clip(channel_image, 0, 255).astype('uint8')
display_grid[col * size : (col + 1) * size, # Displays the grid
row * size : (row + 1) * size] = channel_image
fig_scale = scale / size
fig = plt.figure(figsize=(fig_scale * display_grid.shape[1],
fig_scale * display_grid.shape[0]))
plt.title(layer.name)
plt.grid(False)
plt.imshow(display_grid, aspect='auto', cmap='gray')
if self.save_folder:
asl_detection.save.save(self.save_folder, 'feature_maps', fig=fig)
def visualize_feature_maps(self, index, _layers=1, scale=2):
image = self.images[index:index+1]
self._visualize_feature_maps(image, _layers, scale)
def generate_heat_map(self, _input):
self.grad_cam_names = [layer.name for layer in self.feature_extractor.layers if isinstance(layer, layers.Conv2D)]
image = self.images[_input:_input+1] if isinstance(_input, int) else _input
preds = self.classifier(self.feature_extractor(image))
idx = np.argmax(preds[0])
# initialize our gradient class activation map and build the heatmap
cam = GradCAM(self.feature_extractor, idx, self.grad_cam_names[-1])
heatmap = cam.compute_heatmap(image)
(heatmap, overlay) = cam.overlay_heatmap(heatmap, image, self.img_size, alpha=0.4)
label = self.category_map[idx]
if isinstance(_input, int):
description = 'image\ntrue: {} pred: {}\nconfidence: {:.3f}'.format\
(self.category_map[np.argmax(self.labels[_input])], self.category_map[idx], preds[0][idx])
else:
description = 'pred: {}\nconfidence: {:.3f}'.format(self.category_map[idx], preds[0][idx])
results = {'image': image, 'heatmap': heatmap, 'overlay': overlay, 'description': description, 'label': label}
return results
def visualize_heat_maps(self, index, rows=3, figsize=(8, 8)):
f, axes = plt.subplots(rows, 3, figsize=figsize)
for i in range(rows):
results = self.generate_heat_map(index+i)
axes[i, 0].imshow(results['image'].reshape(self.img_size, self.img_size, 3))
axes[i, 1].imshow(results['heatmap'].reshape(self.img_size, self.img_size, 3))
axes[i, 2].imshow(results['overlay'].reshape(self.img_size, self.img_size, 3))
axes[i, 0].set_title(results['description']).set_size(12)
axes[i, 1].set_title('heatmap')
axes[i, 2].set_title('overlay')
axes[i, 0].axis('off')
axes[i, 1].axis('off')
axes[i, 2].axis('off')
plt.tight_layout(w_pad=0.1)
if self.save_folder:
asl_detection.save.save(self.save_folder, 'heat_maps', fig=f)
# In[8]:
# Traning mobile net for the Dataset
IMG_HEIGHT = 150
IMG_WIDTH = 150
mobilenet = MobileNet(include_top=False,
weights='imagenet',
input_shape=(IMG_HEIGHT, IMG_WIDTH, 3))
output = mobilenet.layers[-1].output
output = tensorflow.keras.layers.Flatten()(output)
mobilenet = Model(mobilenet.input, output)
for layer in mobilenet.layers:
layer.trainable = False
mobilenet.summary()
# In[9]:
input_shape = (IMG_HEIGHT, IMG_WIDTH, 3)
model2 = Sequential()
model2.add(mobilenet)
model2.add(Dense(512, activation='relu', input_dim=input_shape))
model2.add(Dropout(0.3))
model2.add(Dense(512, activation='relu'))
model2.add(Dropout(0.3))
model2.add(Dense(512, activation='relu'))
model2.add(Dropout(0.3))
model2.add(Dense(512, activation='relu'))
model2.add(Dropout(0.3))
def get_model(input_shape,
input_layer,
num_classes,
model_name='resnet_50',
trainable_layers_amount=0,
dropout=0.5,
path_model=None):
if not path_model:
if model_name == 'resnet_50':
base_model = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
else:
if model_name == 'mobile_net':
base_model = MobileNet(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
elif model_name == 'resnet_34':
base_model = ResNet34(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=input_shape)
else:
print(f'Loading model in path {path_model}')
loaded_model = load_model(path_model)
base_model = loaded_model.layers[3]
# Avoid training layers in resnet model.
for layer in base_model.layers:
layer.trainable = False
layers = base_model.layers
# Training the last
if trainable_layers_amount != 0:
if trainable_layers_amount != -1:
trainable_layers = layers[-trainable_layers_amount:]
assert len(trainable_layers) == trainable_layers_amount
else:
trainable_layers = layers
else:
trainable_layers = []
for layer in trainable_layers:
print("Making layer %s trainable " % layer.name)
layer.trainable = True
base_model.summary()
x1 = PreprocessImage(model_name)(input_layer)
x2 = base_model(x1, training=False)
x3 = GlobalAveragePooling2D()(x2)
x4 = Dense(1024, activation='relu')(x3)
x5 = Dropout(dropout)(x4)
output_layer = Dense(
num_classes,
activation='softmax',
name='softmax',
kernel_regularizer=l2(0.01),
bias_regularizer=l2(0.01),
kernel_initializer=tf.keras.initializers.glorot_normal())(x5)
return output_layer, base_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import GlobalMaxPooling2D
from sklearn.decomposition import PCA
from sklearn.manifold.t_sne import TSNE
import os
import glob
import pickle
from pathlib import Path
home = str(Path.home())
mnet = MobileNet(input_shape=(224, 224, 3),
weights='imagenet',
include_top=False)
mnet.summary()
model = GlobalMaxPooling2D()(mnet.layers[-1].output)
model = Model(inputs=mnet.get_input_at(0), outputs=model)
paths = glob.glob(home + '/datasets/mscoco/val2014/*.jpg')
paths = list(paths)
paths.sort()
paths = paths[:2000]
fig = plt.figure(1)
ax1 = fig.add_subplot(121)
ax2 = fig.add_subplot(122)
feats = []
pklpath = '/tmp/feats.pkl'
