class ImgNet(object): def __init__(self, name="resnet"): # 学習済みのVGG16をロード # 構造とともに学習済みの重みも読み込まれる if name == "mobilenet": self.model = MobileNet(weights='imagenet') elif name == "resnet": self.model = ResNet50(weights='imagenet') else: self.model = ResNet50(weights='imagenet') self.model.summary() def array2predict(self, x): # 3次元テンソル(rows, cols, channels) を # 4次元テンソル (samples, rows, cols, channels) に変換 # 入力画像は1枚なのでsamples=1でよい x = np.expand_dims(x, axis=0) # Top-5のクラスを予測する # VGG16の1000クラスはdecode_predictions()で文字列に変換される preds = self.model.predict(preprocess_input(x)) results = decode_predictions(preds, top=5)[0] return results def file2predict(self, filename): # 引数で指定した画像ファイルを読み込む # サイズはVGG16のデフォルトである224x224にリサイズされる img = image.load_img(filename, target_size=(224, 224)) # 読み込んだPIL形式の画像をarrayに変換 x = image.img_to_array(img) results = self.array2predict(x) return results
#!/usr/bin/python # # -*- coding: UTF-8 -*- from tensorflow.keras.preprocessing import image import numpy as np import cv2 from tensorflow.keras.models import load_model from tensorflow.keras.applications.mobilenet import MobileNet from tensorflow.keras.applications.mobilenet import preprocess_input, decode_predictions #include_top=True,完整的模型 #include_top=False,去掉最后的3个全连接层,用来做fine-tuning专用,专门开源了这类模型。 model = MobileNet(weights='imagenet') print(model.summary()) img_path = "elephant.jpg" img = image.load_img(img_path, target_size=(224, 224)) #将输入数据转换为0~1之间 img = image.img_to_array(img) / 255.0 # 为batch添加第四维,axis=0表示在0位置添加,因为MobileNet的Iput层结构是(None,224,224,3) img = np.expand_dims(img, axis=0) print(img.shape) predictions = model.predict(img) print('Predicted:', decode_predictions(predictions, top=3)[0]) print(predictions) description = decode_predictions(predictions, top=3)[0][0][1] src = cv2.imread(img_path) cv2.putText(src, description, (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 0, 0), 2)
# Angkan Biswas # 26.04.2020 # To load pretrained model ''' For the first time model will be downloaded from Keras repository & it will be saved in users home directory. Next time model will be loaded from the users home directory, i.e., /home/dell/.keras/models/''' from tensorflow.keras.applications.vgg16 import VGG16 from tensorflow.keras.applications.mobilenet import MobileNet vgg16Model = VGG16() # Load pretrained model VGG16 [size = 528 MB] vgg16Model.summary() # Display model arcitracture del vgg16Model # Delete loaded model. It is nessary when multiple models need to loaded mobileModel =MobileNet() # Load pretrained mobile [size = 17 MB] mobileModel.summary() # Display model arcitracture
x = tf.keras.layers.Conv2D(num_classes, 1, name='conv_preds')(x) x = tf.keras.layers.Activation('softmax', name='act_softmax')(x) x = tf.keras.layers.Reshape((10,), name='reshape_2')(x) model = tf.keras.Model(inputs=image_input, outputs=x) # ImageNet weights, basic fully connected (dense) top elif option == 3: image_input = tf.keras.Input(shape=x_train.shape[1:], name='image_input') x = MobileNet(weights='imagenet', include_top=False)(image_input) x = tf.keras.layers.Flatten(name='flatten')(x) x = tf.keras.layers.Dense(4096, activation='relu', name='fc1')(x) x = tf.keras.layers.Dense(4096, activation='relu', name='fc2')(x) x = tf.keras.layers.Dense(num_classes, activation='softmax', name='predictions')(x) model = tf.keras.Model(inputs=image_input, outputs=x) model.summary() # Select loss, optimizer, metric model.compile(loss='categorical_crossentropy', optimizer=tf.train.AdamOptimizer(0.001), metrics=['accuracy']) # Train start = timer() model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) # callbacks=[TensorBoardColabCallback(tbc)]
class Classifier: def __init__(self): self.categories = list('ABCDEFGHIJKLMNOPQRSTUVWXYZ') self.category_map = {i: v for i, v in enumerate(self.categories)} self.num_classes = len(self.categories) self.img_size = 224 self.batch_size = 64 self.train_path, self.test_path = None, None self.train_generator, self.test_generator = None, None self.step_size_train, self.step_size_valid = None, None self.images, self.labels = None, None self.classifier = None self.history = None self.grad_cam_names = None self.save_folder = None self.feature_extractor = None self.latent_vectors = None self.latent_path = None self.latent_test = None self.latent_train = None def generate_data(self, train_path, test_path, batch_size, figsize=(10,10), fontsize=16): self.train_path = train_path self.test_path = test_path self.batch_size = batch_size test_datagen = ImageDataGenerator( rescale=1/255.) train_datagen = ImageDataGenerator( rescale=1/255., brightness_range=[.9, 1.], rotation_range=5, zoom_range=.1, width_shift_range=.1, height_shift_range=.1) self.train_generator = train_datagen.flow_from_directory( self.train_path, shuffle=True, target_size=(self.img_size, self.img_size), color_mode='rgb', batch_size=batch_size, seed=0, class_mode="categorical") self.test_generator = test_datagen.flow_from_directory( self.test_path, shuffle=True, target_size=(self.img_size, self.img_size), color_mode='rgb', batch_size=batch_size, seed=0, class_mode="categorical") _, axes = plt.subplots(6, 6, figsize=figsize) for i, category in enumerate(self.categories[:6]): path = self.train_path + '/' + category images = os.listdir(path) for j in range(6): image = cv2.imread(path + '/' + images[j]) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) axes[i, j].imshow(image) axes[i, j].set(xticks=[], yticks=[]) axes[i, j].set_title(category, color = 'tomato').set_size(fontsize) plt.suptitle('Vanilla Data').set_size(2*fontsize) plt.tight_layout() images, labels = self.train_generator.next() _, axes = plt.subplots(6, 6, figsize=figsize) for i in range(6): for j in range(6): image = images[i+j] label = self.category_map[np.argmax(labels[i+j])] axes[i, j].imshow(image) axes[i, j].set(xticks=[], yticks=[]) axes[i, j].set_title(label, color = 'tomato').set_size(fontsize) plt.suptitle('Augmented Data').set_size(2*fontsize) plt.tight_layout() self.step_size_train=int(self.train_generator.n // self.train_generator.batch_size) self.step_size_valid=int(self.test_generator.n // self.test_generator.batch_size) def notify(self, fig): fig.savefig('tmp.jpg') with open('tmp.jpg', 'rb') as f: telegram_send.send(images=[f]) os.remove('tmp.jpg') def plot_accuracy(self, history): f, axes = plt.subplots(1, 2, figsize=(12, 4)) accuracy = history.history['accuracy'] loss = history.history['loss'] val_accuracy = history.history['val_accuracy'] val_loss = history.history['val_loss'] print('Training accuracy: {:.{}f}'.format(np.max(accuracy), 3)) print('Training loss: {:.{}f}'.format(np.max(loss), 3)) print('Validation accuracy: {:.{}f}'.format(np.max(val_accuracy), 3)) print('Validation loss: {:.{}f}'.format(np.max(val_loss), 3)) axes[0].plot(history.history['accuracy']) axes[0].plot(history.history['val_accuracy']) axes[0].set_title('Model accuracy') axes[0].set(ylabel = 'accuracy', xlabel = 'Epoch') axes[0].legend(['Train', 'Test'], loc='upper left') axes[1].plot(history.history['loss']) axes[1].plot(history.history['val_loss']) axes[1].set_title('Model loss') axes[1].set(ylabel = 'Loss', xlabel = 'Epoch') axes[1].legend(['Train', 'Test'], loc='upper left') return f def set_feature_extractor(self, name = 'mobilenet', summary = False): if name == 'mobilenet': self.feature_extractor = MobileNet(input_shape = (self.img_size, self.img_size, 3), include_top=True,weights ='imagenet') output = self.feature_extractor.layers[-6].output self.feature_extractor = tf.keras.Model(self.feature_extractor.inputs, output) if summary: self.feature_extractor.summary() def extract_and_save(self, latent_path, latent_vectors, save=True): '''model: Model used to extract encoded features generator: yields (x_batch, y_batch) ''' self.latent_vectors = latent_vectors self.latent_path = latent_path if not save: return for folder in ['train', 'test']: save_path = os.path.join(latent_path, folder) if os.path.exists(save_path) and os.path.isdir(save_path): shutil.rmtree(save_path) os.makedirs(save_path, exist_ok=True) template = 'batch_{}.h5' batch = 0 for generator in [self.train_generator, self.test_generator]: for x_batch, y_batch in tqdm.tqdm(generator): IPython.display.clear_output(wait=True) features = self.feature_extractor.predict(x_batch) file_path = os.path.join(save_path, template.format(batch)) with h5py.File(file_path, 'w') as file: # encoded features and hard labels file.create_dataset('features', data=features) file.create_dataset('labels', data=y_batch) batch += 1 if folder == 'train': if batch >= self.step_size_train: break else: if batch >= self.step_size_valid: break def load_data(self, folder): '''yields (x_batch, y_batch) for model.fit() ''' root_path = os.path.join(self.latent_path, folder) while True: for file_path in os.listdir(root_path): with h5py.File(os.path.join(root_path, file_path), 'r') as file: yield (np.array(file['features']), np.array(file['labels'])) def clear_session(self): tf.keras.backend.clear_session() def train(self, lr=None, optimizer=None, epochs=None, decay_lr=False, save_folder=None, notification = False): self.save_folder = save_folder # shape of VGG16 encoded features inputs = layers.Input(shape=self.latent_vectors) x = layers.Dense(self.num_classes, activation='softmax')(inputs) self.classifier = tf.keras.Model(inputs, x) self.classifier.compile(optimizer=optimizer(lr=lr), loss='categorical_crossentropy', metrics=['accuracy']) def lr_decay(epoch): alpha, decay = 1, 1 return lr / (alpha + decay * epoch) callback_learning_rate = tf.keras.callbacks.LearningRateScheduler(lr_decay, verbose=True) plot_losses = PlotLosses() callback_is_nan = tf.keras.callbacks.TerminateOnNaN() callback_early = tf.keras.callbacks.EarlyStopping(monitor='loss', min_delta = .001, patience = 10) callbacks = [plot_losses, callback_is_nan, callback_early] callbacks += [callback_learning_rate] if decay_lr else [] self.latent_train, self.latent_test = 'train', 'test' self.history = self.classifier.fit( x = self.load_data(self.latent_train), epochs=epochs, workers=15, steps_per_epoch=self.step_size_train, validation_steps=self.step_size_valid, validation_data=self.load_data(self.latent_test), callbacks=callbacks) fig = self.plot_accuracy(self.history) if save_folder: asl_detection.save.save(save_folder, 'acc_loss', fig=fig) asl_detection.save.save(save_folder, 'model', self.classifier) if notification: self.notify(fig) self.images, self.labels = self.test_generator.next() def _visualize_feature_maps(self, image, _layers, scale): model_layers = self.feature_extractor.layers # Extracts the outputs layer_outputs = [layer.output for layer in self.feature_extractor.layers] # Creates a model that will return these outputs, given the model input activation_model = tf.keras.Model(inputs=self.feature_extractor.inputs, outputs=layer_outputs) # get activations activations = activation_model.predict(image) images_per_row = 4; count = -1 # Displays the feature maps for layer, layer_activation in zip(model_layers, activations): if not isinstance(layer, layers.Conv2D): continue count += 1 # show first 3 conv layers if count != _layers: continue n_features = layer_activation.shape[-1] # Number of features in the feature map size = layer_activation.shape[1] #The feature map has shape (1, size, size, n_features). n_cols = n_features // images_per_row # Tiles the activation channels in this matrix display_grid = np.zeros((size * n_cols, images_per_row * size)) for col in range(n_cols): # Tiles each filter into a big horizontal grid for row in range(images_per_row): channel_image = layer_activation[0, :, :, col * images_per_row + row] # Post-processes the feature to make it visually palatable channel_image -= channel_image.mean() channel_image /= channel_image.std() + 1e-8 channel_image *= 64 channel_image += 128 channel_image = np.clip(channel_image, 0, 255).astype('uint8') display_grid[col * size : (col + 1) * size, # Displays the grid row * size : (row + 1) * size] = channel_image fig_scale = scale / size fig = plt.figure(figsize=(fig_scale * display_grid.shape[1], fig_scale * display_grid.shape[0])) plt.title(layer.name) plt.grid(False) plt.imshow(display_grid, aspect='auto', cmap='gray') if self.save_folder: asl_detection.save.save(self.save_folder, 'feature_maps', fig=fig) def visualize_feature_maps(self, index, _layers=1, scale=2): image = self.images[index:index+1] self._visualize_feature_maps(image, _layers, scale) def generate_heat_map(self, _input): self.grad_cam_names = [layer.name for layer in self.feature_extractor.layers if isinstance(layer, layers.Conv2D)] image = self.images[_input:_input+1] if isinstance(_input, int) else _input preds = self.classifier(self.feature_extractor(image)) idx = np.argmax(preds[0]) # initialize our gradient class activation map and build the heatmap cam = GradCAM(self.feature_extractor, idx, self.grad_cam_names[-1]) heatmap = cam.compute_heatmap(image) (heatmap, overlay) = cam.overlay_heatmap(heatmap, image, self.img_size, alpha=0.4) label = self.category_map[idx] if isinstance(_input, int): description = 'image\ntrue: {} pred: {}\nconfidence: {:.3f}'.format\ (self.category_map[np.argmax(self.labels[_input])], self.category_map[idx], preds[0][idx]) else: description = 'pred: {}\nconfidence: {:.3f}'.format(self.category_map[idx], preds[0][idx]) results = {'image': image, 'heatmap': heatmap, 'overlay': overlay, 'description': description, 'label': label} return results def visualize_heat_maps(self, index, rows=3, figsize=(8, 8)): f, axes = plt.subplots(rows, 3, figsize=figsize) for i in range(rows): results = self.generate_heat_map(index+i) axes[i, 0].imshow(results['image'].reshape(self.img_size, self.img_size, 3)) axes[i, 1].imshow(results['heatmap'].reshape(self.img_size, self.img_size, 3)) axes[i, 2].imshow(results['overlay'].reshape(self.img_size, self.img_size, 3)) axes[i, 0].set_title(results['description']).set_size(12) axes[i, 1].set_title('heatmap') axes[i, 2].set_title('overlay') axes[i, 0].axis('off') axes[i, 1].axis('off') axes[i, 2].axis('off') plt.tight_layout(w_pad=0.1) if self.save_folder: asl_detection.save.save(self.save_folder, 'heat_maps', fig=f)
# In[8]: # Traning mobile net for the Dataset IMG_HEIGHT = 150 IMG_WIDTH = 150 mobilenet = MobileNet(include_top=False, weights='imagenet', input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)) output = mobilenet.layers[-1].output output = tensorflow.keras.layers.Flatten()(output) mobilenet = Model(mobilenet.input, output) for layer in mobilenet.layers: layer.trainable = False mobilenet.summary() # In[9]: input_shape = (IMG_HEIGHT, IMG_WIDTH, 3) model2 = Sequential() model2.add(mobilenet) model2.add(Dense(512, activation='relu', input_dim=input_shape)) model2.add(Dropout(0.3)) model2.add(Dense(512, activation='relu')) model2.add(Dropout(0.3)) model2.add(Dense(512, activation='relu')) model2.add(Dropout(0.3)) model2.add(Dense(512, activation='relu')) model2.add(Dropout(0.3))
def get_model(input_shape, input_layer, num_classes, model_name='resnet_50', trainable_layers_amount=0, dropout=0.5, path_model=None): if not path_model: if model_name == 'resnet_50': base_model = ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape) else: if model_name == 'mobile_net': base_model = MobileNet(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape) elif model_name == 'resnet_34': base_model = ResNet34(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape) else: print(f'Loading model in path {path_model}') loaded_model = load_model(path_model) base_model = loaded_model.layers[3] # Avoid training layers in resnet model. for layer in base_model.layers: layer.trainable = False layers = base_model.layers # Training the last if trainable_layers_amount != 0: if trainable_layers_amount != -1: trainable_layers = layers[-trainable_layers_amount:] assert len(trainable_layers) == trainable_layers_amount else: trainable_layers = layers else: trainable_layers = [] for layer in trainable_layers: print("Making layer %s trainable " % layer.name) layer.trainable = True base_model.summary() x1 = PreprocessImage(model_name)(input_layer) x2 = base_model(x1, training=False) x3 = GlobalAveragePooling2D()(x2) x4 = Dense(1024, activation='relu')(x3) x5 = Dropout(dropout)(x4) output_layer = Dense( num_classes, activation='softmax', name='softmax', kernel_regularizer=l2(0.01), bias_regularizer=l2(0.01), kernel_initializer=tf.keras.initializers.glorot_normal())(x5) return output_layer, base_model
from tensorflow.keras.models import Model from tensorflow.keras.layers import GlobalMaxPooling2D from sklearn.decomposition import PCA from sklearn.manifold.t_sne import TSNE import os import glob import pickle from pathlib import Path home = str(Path.home()) mnet = MobileNet(input_shape=(224, 224, 3), weights='imagenet', include_top=False) mnet.summary() model = GlobalMaxPooling2D()(mnet.layers[-1].output) model = Model(inputs=mnet.get_input_at(0), outputs=model) paths = glob.glob(home + '/datasets/mscoco/val2014/*.jpg') paths = list(paths) paths.sort() paths = paths[:2000] fig = plt.figure(1) ax1 = fig.add_subplot(121) ax2 = fig.add_subplot(122) feats = [] pklpath = '/tmp/feats.pkl'