def init_model(model_name):
if (model_name == "VGG19"): # Initialisierung des VGG19
return VGG19(include_top=True, weights='imagenet')
if (model_name == "VGG16"):
return tf.keras.applications.VGG16(include_top=True,
weights='imagenet')
if (model_name == "ResNet50"):
return ResNet50(include_top=True, weights="imagenet")
if (model_name == "DenseNet201"):
return DenseNet201(include_top=True, weights="imagenet")
if (model_name == "DenseNet121"):
return DenseNet121(include_top=True, weights="imagenet")
if (model_name == "InceptionResNetV2"):
return InceptionResNetV2(include_top=True, weights="imagenet")
def train_raw():
# show class indices
print('****************')
for cls, idx in train_batches.class_indices.items():
print('Class #{} = {}'.format(idx, cls))
print('****************')
# build our classifier model based on pre-trained InceptionResNetV2:
# 1. we don't include the top (fully connected) layers of InceptionResNetV2
# 2. we add a DropOut layer followed by a Dense (fully connected)
# layer which generates softmax class score for each class
# 3. we compile the final model using an Adam optimizer, with a
# low learning rate (since we are 'fine-tuning')
net = InceptionResNetV2(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
x = net.output
x = Flatten()(x)
x = Dropout(0.5)(x)
output_layer = Dense(NUM_CLASSES, activation='softmax', name='softmax')(x)
net_final = Model(inputs=net.input, outputs=output_layer)
for layer in net_final.layers[:FREEZE_LAYERS]:
layer.trainable = False
for layer in net_final.layers[FREEZE_LAYERS:]:
layer.trainable = True
net_final.compile(optimizer=Adam(lr=1e-5),
loss='categorical_crossentropy',
metrics=['accuracy'])
#print(net_final.summary())
# train the model
for i in range(1):
net_final.fit_generator(
train_batches,
steps_per_epoch=train_batches.samples // BATCH_SIZE // 10,
validation_data=valid_batches,
validation_steps=valid_batches.samples // BATCH_SIZE // 10,
epochs=1)
gen_sub(net_final, testdf, sn=i)
WEIGHTS_FINAL = f'./output/model-inception_resnet_v{i}-27.h5'
# save trained weights
net_final.save(WEIGHTS_FINAL)
print(f'weight save to {WEIGHTS_FINAL}')
return WEIGHTS_FINAL
def _load_model(self, model='VGG'):
if model=='VGG':
print("Loading VGG19 pre-trained model...")
base_model = VGG19(weights='imagenet')
base_model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
x = base_model.output
x = GlobalAveragePooling2D()(x)
self.VGG_model = Model(inputs=base_model.input, outputs=x)
if model=='Inception_Resnet':
#load Inception_Resnet model
print("Loading Inception_Resnet_V2 pre-trained model...")
base_model = InceptionResNetV2(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
self.IR_model = Model(inputs=base_model.input, outputs=x)
def get_is_classifier(image_shape, num_of_labels, num_of_classes):
image_input = keras.layers.Input(shape=image_shape)
label_input = keras.layers.Input(shape=(num_of_labels, ))
net = InceptionResNetV2(
include_top=False,
weights=None,
# input_tensor=image_input,
input_shape=image_shape,
pooling='avg')
x = net(image_input)
x = keras.layers.Concatenate()([x, label_input])
x = keras.layers.Dense(num_of_classes,
activation='softmax',
name='predictions')(x)
model = keras.models.Model([image_input, label_input],
x,
name='is_classifier')
return model
def get_model(self, model_name, weights):
if model_name == 'InceptionV3':
from tensorflow.python.keras.applications.inception_v3 import InceptionV3
base_model = InceptionV3(weights=weights, include_top=False)
elif model_name == 'NASNetLarge':
from tensorflow.python.keras.applications.nasnet import NASNetLarge
base_model = NASNetLarge(weights=weights, include_top=False)
elif model_name == 'DenseNet201':
from tensorflow.python.keras.applications.densenet import DenseNet201
base_model = DenseNet201(weights=weights, include_top=False)
elif model_name == 'Xception':
from tensorflow.python.keras.applications.xception import Xception
base_model = Xception(weights=weights, include_top=False)
elif model_name == 'VGG16':
from tensorflow.python.keras.applications.vgg16 import VGG16
base_model = VGG16(weights=weights, include_top=False)
elif model_name == 'VGG19':
from tensorflow.python.keras.applications.vgg19 import VGG19
base_model = VGG19(weights=weights, include_top=False)
elif model_name == 'NASNetMobile':
from tensorflow.python.keras.applications.nasnet import NASNetMobile
base_model = NASNetMobile(weights=weights, include_top=False)
elif model_name == 'MobileNetV2':
from tensorflow.python.keras.applications.mobilenet_v2 import MobileNetV2
base_model = MobileNetV2(weights=weights, include_top=False)
elif model_name == 'ResNet50':
from tensorflow.python.keras.applications.resnet50 import ResNet50
base_model = ResNet50(weights=weights, include_top=False)
elif model_name == 'InceptionResNetV2':
from tensorflow.python.keras.applications.inception_resnet_v2 import InceptionResNetV2
base_model = InceptionResNetV2(weights=weights, include_top=False, )
else:
raise KeyError('Unknown network.')
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(512, activation='relu')(x)
predictions = Dense(1, activation='sigmoid')(x)
model = Model(inputs=base_model.input, outputs=predictions)
return model
def get_model(self, model_name, weights='imagenet'):
if model_name == 'InceptionV3':
from tensorflow.python.keras.applications.inception_v3 import InceptionV3
base_model = InceptionV3(weights=weights, include_top=False)
elif model_name == 'NASNetLarge':
from tensorflow.python.keras.applications.nasnet import NASNetLarge
base_model = NASNetLarge(weights=weights, include_top=False)
elif model_name == 'DenseNet201':
from tensorflow.python.keras.applications.densenet import DenseNet201
base_model = DenseNet201(weights=weights, include_top=False)
elif model_name == 'Xception':
from tensorflow.python.keras.applications.xception import Xception
base_model = Xception(weights=weights, include_top=False)
elif model_name == 'VGG19':
from tensorflow.python.keras.applications.vgg19 import VGG19
base_model = VGG19(weights=weights, include_top=False)
elif model_name == 'NASNetMobile':
from tensorflow.python.keras.applications.nasnet import NASNetMobile
base_model = NASNetMobile(weights=weights, include_top=False)
elif model_name == 'MobileNetV2':
from tensorflow.python.keras.applications.mobilenet_v2 import MobileNetV2
base_model = MobileNetV2(weights=weights, include_top=False)
elif model_name == 'ResNet50':
from tensorflow.python.keras.applications.resnet50 import ResNet50
base_model = ResNet50(weights=weights, include_top=False)
elif model_name == 'InceptionResNetV2':
from tensorflow.python.keras.applications.inception_resnet_v2 import InceptionResNetV2
base_model = InceptionResNetV2(weights=weights, include_top=False)
else:
raise KeyError('Unknown network.')
x = base_model.output
x = GlobalAveragePooling2D()(x)
# x = Dense(1024, activation='relu')(x)
output = Dense(1, activation='sigmoid')(x)
# output = SiameseLossLayer(self.batch_size, self.num_distort, self.num_level)(x)
model = Model(inputs=base_model.input, outputs=output)
self.name = model_name
return model
def create_model(model_name,
input_shape,
target_labels,
n_classes_per_label_type,
n_gpus,
continue_training=False,
rebuild_model=False,
transfer_learning=False,
transfer_learning_type='last_layer',
path_of_model_to_load=None,
initial_learning_rate=0.01,
output_loss_weights=None,
optimizer='sgd'):
""" Returns specified model architecture """
# Load model from disk
if continue_training and not rebuild_model:
logging.debug("Preparing continue_training")
loaded_model = load_model_from_disk(path_of_model_to_load)
return loaded_model
model_input = Input(shape=input_shape, name='images')
if model_name == 'InceptionResNetV2':
keras_model = InceptionResNetV2(include_top=False,
weights=None,
input_tensor=model_input,
input_shape=None,
pooling='avg')
output_flat = keras_model.output
model_input = keras_model.input
elif model_name in [
'ResNet18', 'ResNet34', 'ResNet50', 'ResNet101', 'ResNet152'
]:
res_builder = ResnetBuilder()
if model_name == 'ResNet18':
output_flat = res_builder.build_resnet_18(model_input)
elif model_name == 'ResNet34':
output_flat = res_builder.build_resnet_34(model_input)
elif model_name == 'ResNet50':
output_flat = res_builder.build_resnet_50(model_input)
elif model_name == 'ResNet101':
output_flat = res_builder.build_resnet_101(model_input)
elif model_name == 'ResNet152':
output_flat = res_builder.build_resnet_152(model_input)
elif model_name == 'small_cnn':
output_flat = small_cnn(model_input)
elif model_name == 'Xception':
keras_model = Xception(include_top=False,
weights=None,
input_tensor=model_input,
input_shape=None,
pooling='avg')
output_flat = keras_model.output
model_input = keras_model.input
else:
raise ValueError("Model: %s not implemented" % model_name)
all_target_outputs = list()
for n_classes, target_name in zip(n_classes_per_label_type, target_labels):
all_target_outputs.append(
Dense(units=n_classes,
kernel_initializer="he_normal",
activation='softmax',
name=target_name)(output_flat))
# Define model optimizer
if optimizer == 'sgd':
opt = SGD(lr=initial_learning_rate, momentum=0.9, decay=1e-4)
elif optimizer == 'rmsprop':
opt = RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.0)
else:
raise ValueError("optimizer %s not implemented" % optimizer)
if n_gpus > 1:
logging.debug("Preparing Multi-GPU Model")
with tf.device('/cpu:0'):
base_model = Model(inputs=model_input, outputs=all_target_outputs)
if continue_training and rebuild_model:
logging.debug("Preparing continue_training by \
rebuilding model")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model, base_model, incl_last=True)
elif transfer_learning:
if transfer_learning_type == 'last_layer':
logging.debug("Preparing transfer_learning with freezing \
all but the last layer")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model,
base_model,
incl_last=False)
non_output_layers = get_non_output_layer_ids(base_model)
base_model = set_layers_to_non_trainable(
base_model, non_output_layers)
elif transfer_learning_type == 'all_layers':
logging.debug("Preparing transfer_learning with freezing \
no layers")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model,
base_model,
incl_last=False)
else:
raise ValueError("transfer_learning_type option %s not \
recognized" % transfer_learning)
model = multi_gpu_model(base_model, gpus=n_gpus)
else:
model = Model(inputs=model_input, outputs=all_target_outputs)
if continue_training and rebuild_model:
logging.debug("Preparing continue_training by \
rebuilding model")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model, model, incl_last=True)
elif transfer_learning:
if transfer_learning_type == 'last_layer':
logging.debug("Preparing transfer_learning with freezing \
all but the last layer")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model, model, incl_last=False)
non_output_layers = get_non_output_layer_ids(model)
model = set_layers_to_non_trainable(model, non_output_layers)
elif transfer_learning_type == 'all_layers':
logging.debug("Preparing transfer_learning with freezing \
no layers")
loaded_model = load_model_from_disk(path_of_model_to_load)
copy_model_weights(loaded_model, model, incl_last=False)
else:
raise ValueError("transfer_learning_type option %s not \
recognized" % transfer_learning)
model.compile(loss=build_masked_loss(K.sparse_categorical_crossentropy),
optimizer=opt,
loss_weights=output_loss_weights,
metrics=[accuracy, top_k_accuracy])
return model
#Δήλωση
app = FastAPI()
#Δήλωση μοντέλων μηχανικής μάθησης και οι αντίστοιχες ρυθμίσεις τους
modelRes = ResNet50(weights='imagenet')
modelVGG = VGG16(weights='imagenet', include_top=True)
modelEfficient = EfficientNetB0(include_top=True, weights='imagenet')
modelEfficientB7 = EfficientNetB7(include_top=True, weights='imagenet')
modelNasNet = NASNetLarge(include_top=True, weights='imagenet')
modelMobileNet = MobileNetV2(weights="imagenet", include_top=True)
modelXception = Xception(include_top=True, weights="imagenet")
modelInception = InceptionV3(include_top=True,
weights="imagenet",
classifier_activation="softmax")
modelInRes = InceptionResNetV2(weights="imagenet", include_top=True)
# Settings
MIN_CONFIDENCE = 0.1 # Το ελάχιστο δυνατό confidence που δέχόμαστε απο τα μοντέλα.
# Τα URLs που θα απαντάει το κάθε μοντέλο
IMAGE_URL2 = "/v1/vision/resNet"
IMAGE_URL3 = "/v1/vision/vggNet"
IMAGE_URL4 = "/v1/vision/efficientNet"
IMAGE_URL5 = "/v1/vision/nasNet"
IMAGE_URL6 = "/v1/vision/mobileNet2"
IMAGE_URL7 = "/v1/vision/xceptionNet"
IMAGE_URL8 = "/v1/vision/efficientNetB7"
# Ξεκινώντας στο domain "/" δηλαδή στην αρχική σελίδα του host του server εμφανίζεται το μήνυμα του return
output=base_model.get_layer('avg_pool').output)
image_size = (224, 224)
elif model_name == "inceptionv3":
base_model = InceptionV3(include_top=include_top,
weights=weights,
input_tensor=Input(shape=(299, 299, 3)))
try:
model = Model(base_model.input,
base_model.get_layer('custom').output)
except:
model = Model(input=base_model.input,
output=base_model.get_layer('custom').output)
image_size = (299, 299)
elif model_name == "inceptionresnetv2":
base_model = InceptionResNetV2(include_top=include_top,
weights=weights,
input_tensor=Input(shape=(299, 299, 3)))
try:
model = Model(base_model.input,
base_model.get_layer('custom').output)
except:
model = Model(input=base_model.input,
output=base_model.get_layer('custom').output)
image_size = (299, 299)
elif model_name == "mobilenet":
base_model = MobileNetV2(include_top=include_top,
weights=weights,
input_tensor=Input(shape=(224, 224, 3)),
input_shape=(224, 224, 3))
try:
model = Model(base_model.input,
preserve_range=True)
x_train[nr] = img
nr += 1
x_train = x_train.astype('float32') / 255.
x_train, x_test = train_test_split(x_train,
test_size=TEST_SIZE,
random_state=SEED)
return x_train, x_test
datagen = ImageDataGenerator(shear_range=0.2,
zoom_range=0.2,
rotation_range=20,
horizontal_flip=True)
inception = InceptionResNetV2(weights=None, include_top=True)
inception.load_weights(RESNET_PATH)
inception.graph = get_default_graph()
def create_inception_embedding(grayscaled_rgb):
def resize_gray(x):
return resize(x, (299, 299, 3), mode='constant')
grayscaled_rgb_resized = np.array([resize_gray(x) for x in grayscaled_rgb])
grayscaled_rgb_resized = preprocess_input(grayscaled_rgb_resized)
embed = inception.predict(grayscaled_rgb_resized)
return embed
def image_a_b_gen(dataset):
def fit(self,
learning_rate=1e-4,
epochs=5,
activation='relu',
dropout=0,
hidden_size=1024,
nb_layers=1,
include_class_weight=False,
save_augmented=False,
batch_size=20,
save_model=False,
verbose=True,
fine_tuning=False,
NB_IV3_LAYERS_TO_FREEZE=279,
use_TPU=False,
transfer_model='Inception',
min_accuracy=None,
cutoff_regularization=False,
extract_SavedModel=False):
#if we want stop training when no sufficient improvement in accuracy has been achieved
if min_accuracy is not None:
callback = EarlyStopping(monitor='categorical_accuracy',
baseline=min_accuracy)
callback = [callback]
else:
callback = None
#load the pretrained model, without the classification (top) layers
if transfer_model == 'Xception':
base_model = Xception(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
target_size = (299, 299)
elif transfer_model == 'Inception_Resnet':
base_model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
target_size = (299, 299)
elif transfer_model == 'Resnet':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
target_size = (224, 224)
else:
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
target_size = (299, 299)
#We expect the classes to be the name of the folders in the training set
self.categories = os.listdir(TRAIN_DIR)
#Add the classification layers using Keras functional API
x = base_model.output
x = GlobalAveragePooling2D()(x)
for _ in range(nb_layers):
x = Dense(hidden_size, activation=activation)(
x) #Hidden layer for classification
if dropout > 0:
x = Dropout(rate=dropout)(x)
predictions = Dense(len(self.categories),
activation='softmax')(x) #Output layer
model = Model(inputs=base_model.input, outputs=predictions)
#Set only the top layers as trainable (if we want to do fine-tuning,
#we can train the base layers as a second step)
for layer in base_model.layers:
layer.trainable = False
#Define the optimizer and the loss, and compile the model
loss = 'categorical_crossentropy'
if use_TPU:
#if we want to try out the TPU, it looks like we currently need to use
#tensorflow optimizers...see https://stackoverflow.com/questions/52940552/valueerror-operation-utpu-140462710602256-varisinitializedop-has-been-marked
#...and https://www.youtube.com/watch?v=jgNwywYcH4w
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
tpu_optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
model.compile(optimizer=tpu_optimizer,
loss=loss,
metrics=['categorical_accuracy'])
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
model = tf.contrib.tpu.keras_to_tpu_model(
model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(
TPU_WORKER)))
tf.logging.set_verbosity(tf.logging.INFO)
else:
optimizer = Adam(lr=learning_rate)
model.compile(optimizer=optimizer,
loss=loss,
metrics=['categorical_accuracy'])
datagen_train = ImageDataGenerator(rotation_range=180,
rescale=1. / 255,
width_shift_range=0.1,
height_shift_range=0.1,
shear_range=0.1,
zoom_range=[0.9, 1.5],
horizontal_flip=True,
vertical_flip=True,
fill_mode='nearest')
datagen_val = ImageDataGenerator(rescale=1. / 255)
#Save the augmented images if we want to
if save_augmented:
save_to_dir = AUGMENTED_DIR
else:
save_to_dir = None
self.generator_train = datagen_train.flow_from_directory(
directory=TRAIN_DIR,
target_size=target_size,
batch_size=batch_size,
shuffle=True,
save_to_dir=save_to_dir)
self.generator_val = datagen_val.flow_from_directory(
directory=VAL_DIR,
target_size=target_size,
batch_size=batch_size,
shuffle=False)
steps_per_epoch = self.generator_train.n / batch_size
self.val_steps_per_epoch = self.generator_val.n / batch_size
#if we want to weight the classes given the imbalanced number of images
if include_class_weight:
from sklearn.utils.class_weight import compute_class_weight
cls_train = self.generator_train.classes
class_weight = compute_class_weight(class_weight='balanced',
classes=np.unique(cls_train),
y=cls_train)
else:
class_weight = None
#Fit the model
history = model.fit_generator(
generator=self.generator_train,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
verbose=verbose,
class_weight=class_weight,
validation_data=self.generator_val,
validation_steps=self.val_steps_per_epoch,
callbacks=callback)
#Fine-tune the model, if we wish so
if fine_tuning and not model.stop_training:
print('============')
print('Begin fine-tuning')
print('============')
#declare the first layers as trainable
for layer in model.layers[:NB_IV3_LAYERS_TO_FREEZE]:
layer.trainable = False
for layer in model.layers[NB_IV3_LAYERS_TO_FREEZE:]:
layer.trainable = True
model.compile(optimizer=Adam(lr=learning_rate * 0.1),
loss=loss,
metrics=['categorical_accuracy'])
#Fit the model
history = model.fit_generator(
generator=self.generator_train,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
verbose=verbose,
class_weight=class_weight,
validation_data=self.generator_val,
validation_steps=self.val_steps_per_epoch)
#Evaluate the model, just to be sure
self.fitness = history.history['val_categorical_accuracy'][-1]
#Save the model
if save_model:
if not os.path.exists(parentdir + '/data/trained_models'):
os.makedirs(parentdir + '/data/trained_models')
model.save(parentdir + '/data/trained_models/trained_model.h5')
print('Model saved!')
#save model in production format
if extract_SavedModel:
export_path = "./image_classifier/1/"
with K.get_session() as sess:
tf.saved_model.simple_save(
sess,
export_path,
inputs={'input_image': model.input},
outputs={t.name: t
for t in model.outputs})
else:
self.model = model
del history
del model
def fit(self,
learning_rate=1e-4,
epochs=5,
activation='relu',
dropout=0,
hidden_size=1024,
nb_layers=1,
include_class_weight=False,
batch_size=20,
save_model=False,
verbose=True,
fine_tuning=False,
NB_IV3_LAYERS_TO_FREEZE=279,
use_TPU=False,
transfer_model='Inception',
min_accuracy=None,
extract_SavedModel=False):
#read the tfrecords data
TRAIN_DATA = tf.data.TFRecordDataset(['train.tfrecord'])
VAL_DATA = tf.data.TFRecordDataset(['val.tfrecord'])
print('Read the TFrecords')
if transfer_model in ['Inception', 'Xception', 'Inception_Resnet']:
target_size = (299, 299)
else:
target_size = (224, 224)
#We expect the classes to be the name of the folders in the training set
self.categories = os.listdir(TRAIN_DIR)
"""
helper functions to load tfrecords. Strongly inspired by
https://colab.research.google.com/github/GoogleCloudPlatform/training-data-analyst/blob/master/courses/fast-and-lean-data-science/07_Keras_Flowers_TPU_playground.ipynb#scrollTo=LtAVr-4CP1rp
"""
def read_tfrecord(example):
features = {
"image": tf.FixedLenFeature(
(), tf.string), # tf.string means byte string
"label": tf.FixedLenFeature((), tf.int64)
}
example = tf.parse_single_example(example, features)
image = tf.image.decode_jpeg(example['image'])
image = tf.cast(
image,
tf.float32) / 255.0 # convert image to floats in [0, 1] range
image = tf.image.resize_images(
image,
size=[*target_size],
method=tf.image.ResizeMethod.BILINEAR)
feature = tf.reshape(image, [*target_size, 3])
label = tf.cast(example['label'], tf.int32) # byte string
target = tf.one_hot(label, len(self.categories))
return feature, target
def get_training_dataset():
dataset = TRAIN_DATA.map(read_tfrecord)
dataset = dataset.cache()
dataset = dataset.repeat()
dataset = dataset.shuffle(1000)
dataset = dataset.batch(
batch_size,
drop_remainder=True) # drop_remainder needed on TPU
dataset = dataset.prefetch(
-1
) # prefetch next batch while training (-1: autotune prefetch buffer size)
return dataset
def get_validation_dataset():
dataset = VAL_DATA.map(read_tfrecord)
dataset = dataset.cache()
dataset = dataset.repeat()
dataset = dataset.shuffle(1000)
dataset = dataset.batch(
batch_size,
drop_remainder=True) # drop_remainder needed on TPU
dataset = dataset.prefetch(
-1
) # prefetch next batch while training (-1: autotune prefetch buffer size)
return dataset
#if we want stop training when no sufficient improvement in accuracy has been achieved
if min_accuracy is not None:
callback = EarlyStopping(monitor='categorical_accuracy',
baseline=min_accuracy)
callback = [callback]
else:
callback = None
#load the pretrained model, without the classification (top) layers
if transfer_model == 'Xception':
base_model = Xception(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
elif transfer_model == 'Inception_Resnet':
base_model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
elif transfer_model == 'Resnet':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
else:
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
#Add the classification layers using Keras functional API
x = base_model.output
x = GlobalAveragePooling2D()(x)
for _ in range(nb_layers):
x = Dense(hidden_size, activation=activation)(
x) #Hidden layer for classification
if dropout > 0:
x = Dropout(rate=dropout)(x)
predictions = Dense(len(self.categories),
activation='softmax')(x) #Output layer
model = Model(inputs=base_model.input, outputs=predictions)
#Set only the top layers as trainable (if we want to do fine-tuning,
#we can train the base layers as a second step)
for layer in base_model.layers:
layer.trainable = False
#Define the optimizer and the loss, and compile the model
loss = 'categorical_crossentropy'
if use_TPU:
#if we want to try out the TPU, it looks like we currently need to use
#tensorflow optimizers...see https://stackoverflow.com/questions/52940552/valueerror-operation-utpu-140462710602256-varisinitializedop-has-been-marked
#...and https://www.youtube.com/watch?v=jgNwywYcH4w
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
tpu_optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
model.compile(optimizer=tpu_optimizer,
loss=loss,
metrics=['categorical_accuracy'])
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
model = tf.contrib.tpu.keras_to_tpu_model(
model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(
TPU_WORKER)))
tf.logging.set_verbosity(tf.logging.INFO)
else:
optimizer = Adam(lr=learning_rate)
model.compile(optimizer=optimizer,
loss=loss,
metrics=['categorical_accuracy'])
#if we want to weight the classes given the imbalanced number of images
if include_class_weight:
from sklearn.utils.class_weight import compute_class_weight
cls_train = self.categories
class_weight = compute_class_weight(class_weight='balanced',
classes=np.unique(cls_train),
y=cls_train)
else:
class_weight = None
steps_per_epoch = int(
sum([
len(files)
for r, d, files in os.walk(parentdir +
'/data/image_dataset/train')
]) / batch_size)
validation_steps = int(
sum([
len(files)
for r, d, files in os.walk(parentdir +
'/data/image_dataset/val')
]) / batch_size)
#Fit the model
if use_TPU:
history = model.fit(get_training_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset,
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
else:
history = model.fit(get_training_dataset(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset(),
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
#Fine-tune the model, if we wish so
if fine_tuning and not model.stop_training:
print('============')
print('Begin fine-tuning')
print('============')
#declare the first layers as trainable
for layer in model.layers[:NB_IV3_LAYERS_TO_FREEZE]:
layer.trainable = False
for layer in model.layers[NB_IV3_LAYERS_TO_FREEZE:]:
layer.trainable = True
model.compile(optimizer=Adam(lr=learning_rate * 0.1),
loss=loss,
metrics=['categorical_accuracy'])
#Fit the model
if use_TPU:
history = model.fit(get_training_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset,
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
else:
history = model.fit(get_training_dataset(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset(),
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
#Evaluate the model, just to be sure
self.fitness = history.history['val_categorical_accuracy'][-1]
#Save the model
if save_model:
if not os.path.exists(parentdir + '/data/trained_models'):
os.makedirs(parentdir + '/data/trained_models')
model.save(parentdir + '/data/trained_models/trained_model.h5')
print('Model saved!')
#save model in production format
if extract_SavedModel:
export_path = "./image_classifier/1/"
with K.get_session() as sess:
tf.saved_model.simple_save(
sess,
export_path,
inputs={'input_image': model.input},
outputs={t.name: t
for t in model.outputs})
else:
self.model = model
del history
del model
import tensorflow as tf
from tensorflow.python import keras
from tensorflow.python.keras.applications.vgg16 import VGG16
from tensorflow.python.keras.applications.vgg19 import VGG19
from tensorflow.python.keras.applications.densenet import DenseNet121
from tensorflow.python.keras.applications.densenet import DenseNet201
from tensorflow.python.keras.applications.resnet50 import ResNet50
from tensorflow.python.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.python.keras.models import Model
# Für jedes Modell wird die Summary ausgegeben
print("=== VGG 16 ===")
VGG16().summary()
print("=== VGG 19 ===")
VGG19().summary()
print("=== ResNet50 ===")
ResNet50().summary()
print("=== DenseNet121 ===")
DenseNet121().summary()
print("=== DenseNet201 ===")
DenseNet201().summary()
print("=== InceptionResNetV2 ===")
InceptionResNetV2().summary()
#
# hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# hist = cv2.calcHist([hsv], [0, 1], None, dim, [0, dim[0], 0, dim[1]])
#
# hist = cv.calcHist([img], [0], None, [256], [0, 256])
# clf = SVC(random_state=9)
# transformer = ColorHistTransformer()
model = Model3()
# Pipeline(
# [('hist_color_transformer', transformer),
# ('clf', clf),
# ])
for i in [30000]: #, 10, 100, 1000]:
generated = test_photos_bw
inception = InceptionResNetV2(weights='imagenet', include_top=True)
#inception.graph = get_default_graph()
#inception.load_weights('./data/inception_resnet_v2_weights_tf_dim_ordering_tf_kernels.h5')
def create_inception_embedding(grayscaled_rgb):
grayscaled_rgb_resized = []
for i in grayscaled_rgb:
i = cv2.resize(i, (299, 299), interpolation=cv2.INTER_AREA)
#grayscaled_rgb_resized.append( np.dstack((i,i,i)))
grayscaled_rgb_resized.append(
cv2.cvtColor(i, cv2.COLOR_GRAY2RGB))
#cv2.imwrite("img_train_{}.png".format(0), grayscaled_rgb_resized)
grayscaled_rgb_resized = preprocess_input(
np.array(grayscaled_rgb_resized))
def _calculate_features(self, data_augmentation=False):
"""
data_augmentation: if True, we calculate the features for the augmented dataset too
"""
def calculate_features(model, preprocessor, img, transformation):
"""
transformation: type of transformation to perform data augmentation
000: no transformation
0001: left-right flip
0002: up-down flip
00090: 90d rotation
000180: 180d rotation
000270: 270d rotation
"""
#preprocess the image
img = image.img_to_array(img) # convert to array
#flip
if transformation == '0001':
img = np.fliplr(img)
elif transformation == '0002':
img = np.flipud(img)
#rotate
elif transformation == '00090':
img = rotate(img, angle=90)
elif transformation == '000180':
img = rotate(img, angle=180)
elif transformation == '000270':
img = rotate(img, angle=270)
img = np.expand_dims(img, axis=0)
img = preprocessor(img)
return model.predict(img).flatten()
if data_augmentation:
transformations = [
'000', '0001', '0002', '00090', '000180', '000270'
]
else:
transformations = ['000']
#load VGG19 model
print("Loading VGG19 pre-trained model...")
base_model = VGG19(weights='imagenet')
base_model = Model(inputs=base_model.input,
outputs=base_model.get_layer('block4_pool').output)
x = base_model.output
x = GlobalAveragePooling2D()(x)
VGG_model = Model(inputs=base_model.input, outputs=x)
#load Inception_Resnet model
print("Loading Inception_Resnet_V2 pre-trained model...")
base_model = InceptionResNetV2(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
IR_model = Model(inputs=base_model.input, outputs=x)
#connect to the database, and create the features table if it does not exists
os.makedirs(parentdir + '\\data\\database', exist_ok=True)
conn = sqlite3.connect(parentdir + '\\data\\database\\features.db',
detect_types=sqlite3.PARSE_DECLTYPES)
cur = conn.cursor()
cur.execute(
'CREATE TABLE IF NOT EXISTS features_' + str(self.dataset) +
' (img_id TEXT PRIMARY KEY, item_id TEXT, features_VGG array, features_Inception_Resnet array, transformation CHARACTER(20), white_background INTEGER, active INTEGER)'
)
#create a item ID: list of associated image IDs dictionary, useful to identify most similar items after computation
folder = parentdir + '\\data\\dataset\\' + self.dataset
#extract the id of the items and of the images in the dataset
images = os.listdir(folder)
items = [i.split('_')[0].split('.')[0] for i in images]
self.item_to_img = {
items[i]:
[j for j in images if j.split('_')[0].split('.')[0] == items[i]]
for i in range(len(items))
} #dictionary item ID: img ID
self.img_to_item = {i: i.split('_')[0].split('.')[0] for i in images}
#loop through the images, to extract their features.
cur.execute('UPDATE features_' + str(self.dataset) + ' SET active = ?',
(0, ))
ki = 0
for i in images:
img_ids = [i + ',' + j for j in transformations]
cur.execute(
'SELECT img_id, item_id FROM features_' + str(self.dataset) +
' WHERE img_id IN ({})'.format(','.join(
'?' * len(transformations))), img_ids)
data = cur.fetchall()
path = folder + '\\' + i
img_VGG = image.load_img(path, target_size=(224, 224))
img_IR = image.load_img(path, target_size=(299, 299))
for j in range(len(transformations)):
#if already calculated, we activate it
if img_ids[j] in [x[0] for x in data]:
cur.execute(
'UPDATE features_' + str(self.dataset) +
' SET active = ? WHERE img_id = ?', (1, img_ids[j]))
#otherwise, we calculate it
else:
#VGG model
features_VGG = calculate_features(
model=VGG_model,
preprocessor=ppVGG19,
img=img_VGG,
transformation=transformations[j])
#Inception_Resnet model
features_IR = calculate_features(
model=IR_model,
preprocessor=ppIR,
img=img_IR,
transformation=transformations[j])
#Verify color of the background (if white or not)
if np.array(img_VGG)[0][0][0] == 255:
white_background = 1
else:
white_background = 0
cur.execute(
'INSERT INTO features_' + str(self.dataset) +
' (img_id, item_id, features_VGG, features_Inception_Resnet, transformation, white_background, active) VALUES (?,?,?,?,?,?,?)',
(img_ids[j], i.split('_')[0].split('.')[0],
features_VGG, features_IR, transformations[j],
white_background, 1))
ki += 1
if ki % 100 == 1:
#commit changes
conn.commit()
print('Features known or calculated for', ki, 'images')
conn.commit()
cur.close()
conn.close()
labels.append(ll)
images.append(image)
images = np.array(images)
labels = np.array(labels)
x_train, x_test, y_train, y_test = train_test_split(images,
labels,
test_size=0.1)
# y_train = np_utils.to_categorical(y_train, num_classes)
# y_test = np_utils.to_categorical(y_test, num_classes)
# Cargar modelo preentrenado
inp = Input(shape=img_shape_full)
model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=img_shape_full,
input_tensor=inp)
# Anadir capa para nuestro numero de clases
x = model.output
x = GlobalAveragePooling2D()(x)
predictions = Dense(num_classes, activation='softmax')(x)
model = Model(inputs=model.input, outputs=predictions)
# TOTAL CAPAS = 782
LAYERS_TO_FREEZE = 700
for layer in model.layers[:LAYERS_TO_FREEZE]:
layer.trainable = False
model.compile(optimizer="adam",
loss='categorical_crossentropy',
def create_model(self):
"""
https://github.com/tensorflow/tensorflow/issues/14356
"""
input_shape = (self.config.tfr_image_height,
self.config.tfr_image_width,
self.config.tfr_image_channels)
## VGG16
if self.config.model_name == 'vgg16':
base_model = VGG16(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
## Xception
elif self.config.model_name == 'xception':
base_model = Xception(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
## Resnet50
elif self.config.model_name == 'resnet50':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
logits = self.model_top_resnet50(base_model)
## InceptionResNetV2
elif self.config.model_name == 'inception_resnet_v2':
base_model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
## Densenet121
elif self.config.model_name == 'densenet121':
base_model = DenseNet121(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
logits = self.model_top_densenet121(base_model)
## Densenet169
elif self.config.model_name == 'densenet169':
base_model = DenseNet169(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
logits = self.model_top_densenet121(base_model)
## Densenet201
elif self.config.model_name == 'densenet201':
base_model = DenseNet201(weights='imagenet',
include_top=False,
input_tensor=self.features,
input_shape=input_shape)
logits = self.model_top_densenet121(base_model)
else:
logging.error('Unknown model_name {}'.format(model_name))
exit(1)
return logits
# show class indices
print('****************')
for cls, idx in train_batches.class_indices.items():
print('Class #{} = {}'.format(idx, cls))
print('****************')
# In[15]:
# build our classifier model based on pre-trained InceptionResNetV2:
# 1. we don't include the top (fully connected) layers of InceptionResNetV2
# 2. we add a DropOut layer followed by a Dense (fully connected)
# layer which generates softmax class score for each class
# 3. we compile the final model using an Adam optimizer, with a
# low learning rate (since we are 'fine-tuning')
net = InceptionResNetV2(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=(IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
x = net.output
x = Flatten()(x)
x = Dropout(0.5)(x)
output_layer = Dense(NUM_CLASSES, activation='softmax', name='softmax')(x)
net_final = Model(inputs=net.input, outputs=output_layer)
for layer in net_final.layers[:FREEZE_LAYERS]:
layer.trainable = False
for layer in net_final.layers[FREEZE_LAYERS:]:
layer.trainable = True
net_final.compile(optimizer=tf.keras.optimizers.Adam(lr=1e-5),
loss='categorical_crossentropy',
metrics=['accuracy'])
#print(net_final.summary())
def fit(self,
learning_rate=1e-4,
epochs=5,
activation='relu',
dropout=0,
hidden_size=1024,
nb_layers=1,
include_class_weight=False,
batch_size=20,
save_model=False,
verbose=True,
fine_tuning=False,
NB_IV3_LAYERS_TO_FREEZE=279,
use_TPU=False,
transfer_model='Inception',
min_accuracy=None,
extract_SavedModel=False):
if transfer_model in ['Inception', 'Xception', 'Inception_Resnet']:
target_size = (299, 299)
else:
target_size = (224, 224)
#We expect the classes to be the name of the folders in the training set
self.categories = os.listdir(TRAIN_DIR)
"""
helper functions to to build tensors
inspired by https://www.tensorflow.org/tutorials/load_data/images
"""
def prepare_image(img_path):
#reshape the image
image = Image.open(img_path)
image = image.resize(target_size,
PIL.Image.BILINEAR).convert("RGB")
#convert the image into a numpy array, and expend to a size 4 tensor
image = img_to_array(image)
#rescale the pixels to a 0-1 range
image = image.astype(np.float32) / 255
return image
def generate_tuples(img_folder):
#loop through all the images
# Get all file names of images present in folder
classes = os.listdir(img_folder)
classes_paths = [
os.path.abspath(os.path.join(img_folder, i)) for i in classes
]
x = []
y = []
for i, j in enumerate(classes):
#for all the classes, get the list of pictures
img_paths = os.listdir(classes_paths[i])
img_paths = [
os.path.abspath(os.path.join(classes_paths[i], x))
for x in img_paths
]
for img_path in img_paths:
x.append(prepare_image(img_path))
y = y + [i]
return (np.array(x), np.array(y).astype(np.int32))
#get training data
(x_train,
y_train) = generate_tuples(parentdir + '/data/image_dataset/train')
(x_val, y_val) = generate_tuples(parentdir + '/data/image_dataset/val')
#train input_function: see https://colab.research.google.com/drive/1F8txK1JLXKtAkcvSRQz2o7NSTNoksuU2#scrollTo=abbwQQfH0td3
def get_training_dataset(batch_size=batch_size):
# Convert the inputs to a Dataset.
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
# Shuffle, repeat, and batch the examples.
dataset = dataset.shuffle(1000).repeat().batch(batch_size,
drop_remainder=True)
return dataset
def get_validation_dataset(batch_size=batch_size):
# Convert the inputs to a Dataset.
dataset = tf.data.Dataset.from_tensor_slices((x_val, y_val))
# Shuffle, repeat, and batch the examples.
dataset = dataset.shuffle(1000).repeat().batch(batch_size,
drop_remainder=True)
return dataset
#if we want stop training when no sufficient improvement in accuracy has been achieved
if min_accuracy is not None:
callback = EarlyStopping(monitor='acc', baseline=min_accuracy)
callback = [callback]
else:
callback = None
#load the pretrained model, without the classification (top) layers
if transfer_model == 'Xception':
base_model = Xception(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
elif transfer_model == 'Inception_Resnet':
base_model = InceptionResNetV2(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
elif transfer_model == 'Resnet':
base_model = ResNet50(weights='imagenet',
include_top=False,
input_shape=(224, 224, 3))
else:
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
#Add the classification layers using Keras functional API
x = base_model.output
x = GlobalAveragePooling2D()(x)
for _ in range(nb_layers):
x = Dense(hidden_size, activation=activation)(
x) #Hidden layer for classification
if dropout > 0:
x = Dropout(rate=dropout)(x)
predictions = Dense(len(self.categories),
activation='softmax')(x) #Output layer
model = Model(inputs=base_model.input, outputs=predictions)
#Set only the top layers as trainable (if we want to do fine-tuning,
#we can train the base layers as a second step)
for layer in base_model.layers:
layer.trainable = False
#Define the optimizer and the loss, and compile the model
loss = 'sparse_categorical_crossentropy'
if use_TPU:
#if we want to try out the TPU, it looks like we currently need to use
#tensorflow optimizers...see https://stackoverflow.com/questions/52940552/valueerror-operation-utpu-140462710602256-varisinitializedop-has-been-marked
#...and https://www.youtube.com/watch?v=jgNwywYcH4w
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
model.compile(optimizer=optimizer,
loss=sparse_softmax_cross_entropy,
metrics=['acc'])
TPU_WORKER = 'grpc://' + os.environ['COLAB_TPU_ADDR']
model = tf.contrib.tpu.keras_to_tpu_model(
model,
strategy=tf.contrib.tpu.TPUDistributionStrategy(
tf.contrib.cluster_resolver.TPUClusterResolver(
TPU_WORKER)))
tf.logging.set_verbosity(tf.logging.INFO)
else:
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
model.compile(optimizer=optimizer, loss=loss, metrics=['acc'])
#if we want to weight the classes given the imbalanced number of images
if include_class_weight:
from sklearn.utils.class_weight import compute_class_weight
cls_train = self.categories
class_weight = compute_class_weight(class_weight='balanced',
classes=np.unique(cls_train),
y=cls_train)
else:
class_weight = None
steps_per_epoch = int(
sum([
len(files)
for r, d, files in os.walk(parentdir +
'/data/image_dataset/train')
]) / batch_size)
validation_steps = int(
sum([
len(files)
for r, d, files in os.walk(parentdir +
'/data/image_dataset/val')
]) / batch_size)
#Fit the model
if use_TPU:
history = model.fit(get_training_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset,
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
else:
history = model.fit(get_training_dataset(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset(),
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
#Fine-tune the model, if we wish so
if fine_tuning and not model.stop_training:
print('============')
print('Begin fine-tuning')
print('============')
#declare the first layers as trainable
for layer in model.layers[:NB_IV3_LAYERS_TO_FREEZE]:
layer.trainable = False
for layer in model.layers[NB_IV3_LAYERS_TO_FREEZE:]:
layer.trainable = True
model.compile(optimizer=tf.train.AdamOptimizer(
learning_rate=learning_rate * 0.1),
loss=loss,
metrics=['acc'])
#Fit the model
if use_TPU:
history = model.fit(get_training_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset,
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
else:
history = model.fit(get_training_dataset(),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=get_validation_dataset(),
validation_steps=validation_steps,
verbose=verbose,
callbacks=callback,
class_weight=class_weight)
#Evaluate the model, just to be sure
self.fitness = history.history['val_categorical_accuracy'][-1]
#Save the model
if save_model:
if not os.path.exists(parentdir + '/data/trained_models'):
os.makedirs(parentdir + '/data/trained_models')
model.save(parentdir + '/data/trained_models/trained_model.h5')
print('Model saved!')
#save model in production format
if extract_SavedModel:
export_path = "./image_classifier/1/"
with K.get_session() as sess:
tf.saved_model.simple_save(
sess,
export_path,
inputs={'input_image': model.input},
outputs={t.name: t
for t in model.outputs})
else:
self.model = model
del history
del model
rotation_range=40,
width_shift_range=0.3,
height_shift_range=0.3,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True,
vertical_flip=True
)
validation_datagen = ImageDataGenerator()
train_generator = train_datagen.flow_from_directory(train_dir, batch_size=20, class_mode='binary',
target_size=(224, 224))
validation_generator = validation_datagen.flow_from_directory(validation_dir, batch_size=20, class_mode='binary',
target_size=(224, 224))
net_base = InceptionResNetV2(weights='imagenet', include_top=False, input_shape=(224, 224, 3))
# net_base = efn.EfficientNetB0(weights='imagenet', include_top=False)
flag = False
if (flag):
net_base.trainable = True
set_trainable = False
for layer in net_base.layers:
if layer.name == "block8_10_conv":
set_trainable = True
if set_trainable:
layer.trainable = True
else:
layer.trainable = False