def download_mobilenet_model():
""" Download mobilenet model from keras
"""
print('Downloading MobileNet keras model')
model = MobileNet(weights='imagenet', input_tensor=Input(shape=(224, 224, 3)))
model.save('keras_models/mobilenet.h5')
print('Finished Downloading MobileNet keras model')
class MobileNetModel: #(ClassificationModel):
def __init__(self):
#super(MobileNetModel, self).__init___(model_name='MobileNet')
self.num_classes = 2
self.build_model()
return
def build_model(self):
# Initializing the model with random wights
self.arch = MobileNet(weights=None,
input_shape=(256, 256, 3),
classes=self.num_classes)
# Compiling model with optimization of RSM and cross entropy loss
self.arch.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
return
def __repr__(self):
return str(self.arch.summary())
def fit_data(self,
train_images,
train_labels,
val_images,
val_labels,
initial_epoch=None):
train_history = self.arch.fit(train_images,
train_labels,
epochs=5,
steps_per_epoch=train_images.shape[0],
validation_steps=val_images.shape[0],
validation_data=(val_images, val_labels),
shuffle=True)
return train_history
def save_model(self, model_path):
self.arch.save(model_path)
return
def load_model(self, model_path):
self.arch = load_model(model_path)
return
def test_template_dl_keras(self):
self.assertEqual(restapi_version(), "0.1.1237")
temp = get_temp_folder(__file__, "temp_template_dl_keras")
from keras.applications.mobilenet import MobileNet # pylint: disable=E0401,E0611
model = MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
model_name = os.path.join(temp, "model.keras")
model.save(model_name)
img_input = os.path.join(temp, "..", "data", "wiki_modified2.png")
img_input = numpy.array(Image.open(img_input))
mo = restapi_load({'model': model_name})
pred = restapi_predict(mo, img_input)
self.assertIsInstance(pred, numpy.ndarray)
self.assertEqual(pred.shape, (1, 1000))
def main(args):
train_data = args.train_data
valid_data = args.valid_data
nb_train_samples = args.train_samples
nb_validation_samples = args.valid_samples
model_path = args.model_path
model_weights_path = args.model_weights_path
num_classes = args.num_classes
batch_size = args.batch_size
epochs = args.epochs
loss = args.loss
optimizer = args.optimizer
img_width, img_height = args.target_size
if not os.path.isdir(train_data):
raise EnvironmentError(f'--train-data {train_data} should exist')
if not os.path.isdir(valid_data):
raise EnvironmentError(f'--valid-data {valid_data} should exist')
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
model_params = {
'input_tensor': Input(shape=input_shape),
'classes': num_classes,
'weights': None,
}
print(
f'Start training mobile net for {epochs} epochs.',
f'{nb_train_samples} train samples, {nb_validation_samples} valid samples'
)
model = MobileNet(**model_params)
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
train_datagen = ImageDataGenerator(rescale=1. / 255)
test_datagen = ImageDataGenerator(rescale=1. / 255)
train_generator = train_datagen.flow_from_directory(
train_data,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='categorical',
)
validation_generator = test_datagen.flow_from_directory(
valid_data,
target_size=(img_width, img_height),
batch_size=batch_size,
class_mode='categorical',
)
model.fit_generator(train_generator,
steps_per_epoch=nb_train_samples // batch_size,
epochs=epochs,
validation_data=validation_generator,
validation_steps=nb_validation_samples // batch_size,
verbose=1)
model.save(model_path)
model.save_weights(model_weights_path)
print('Model saved.')
return 0
# Train on ImageNet
checkpoint_path = "Mobilenet/cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
os.makedirs(checkpoint_dir, exist_ok=True)
cp_callback = keras.callbacks.ModelCheckpoint(
checkpoint_path,
verbose=1,
save_weights_only=True,
# Save weights, every 1-epoch
period=1)
csv_logger = keras.callbacks.CSVLogger('MobileNet_training.csv')
print("Starting to train Modified MobileNet...")
epochs = 5
model.fit_generator(imagenet_generator(train_dataset,
batch_size=Nbatch,
is_training=True),
steps_per_epoch=Ntrain // Nbatch,
epochs=epochs,
validation_data=imagenet_generator(validation_dataset,
batch_size=Nbatch),
validation_steps=Nvalidation // Nbatch,
verbose=1,
callbacks=[cp_callback, csv_logger])
model.save("MobileNet.h5")
# mse = losses.mean_squared_error
adam = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=adam,
metrics=['accuracy'])
# Fit the model
# f = open("output/log.csv","w+")
# csv_logger = CSVLogger('/output/log.csv', append=True, separator=',')
tb_callback = TensorBoard(log_dir='/output/logs',
histogram_freq=0,
batch_size=batch_size)
# model.fit(train_features, train_labels, epochs=128, batch_size=batch_size, verbose=2, callbacks=[csv_logger])
model.fit_generator(train_generator,
epochs=150,
steps_per_epoch=(1400 / batch_size) + 1,
verbose=2,
callbacks=[tb_callback])
model.save("/output/mobnet.h5")
score, acc = model.evaluate_generator(test_generator,
steps=(550 / batch_size) + 1)
# calculate predictions
# pred = model.predict(test_features)
# print(test_labels.shape,pred.shape)
# print(test_labels[0],pred[0])
target_names = ['blade', 'gun', 'others', 'shuriken']
print("Test score: " + str(score))
print("Test accuracy: " + str(acc))
# print(classification_report(test_labels, pred,target_names=target_names))
import numpy as np
from keras.models import load_model
from keras.applications.mobilenet import MobileNet
# Mobile Net
model = MobileNet(weights="imagenet")
# モデルの保存、モデルをロードした後予測しかしないため、include_optimizer=Falseとする
model.save('mobile_net_model.h5', include_optimizer=False)
exit(1)
model.summary()
print("TRAINING PHASE")
decay = lrate / epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
tensorboard = TensorBoard(log_dir=logdir, histogram_freq=0, write_graph=True)
tensorboard.set_model(model)
history = model.fit_generator(train_generator,
epochs=epochs,
validation_data=validation_generator,
callbacks=[tensorboard])
model.save("leaf.%s.h5" % runmodel)
print("TESTING PHASE")
evaltest = model.evaluate_generator(test_generator, 1)
for name, val in zip(model.metrics_names, evaltest):
print(name, val)
print("END", datetime.datetime.now().isoformat())
input_tensor=inputs,
pooling='max')
# freeze the network
#for layer in model.layers:
# layer.trainable = False
x = model.output
pred = Dense(3, activation='softmax')(x)
model = Model(model.input, pred)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
tb_cb = keras.callbacks.TensorBoard(log_dir='tensorboard-log', batch_size=batch_size)
cbks = [tb_cb]
# train model
print("Start training model {}".format(model_name))
history = model.fit(x=X, y=y,
batch_size=batch_size, epochs=args.epochs,
validation_split=0.12,
shuffle=True,
callbacks=cbks)
# save model
model.save(model_name + ".h5")
print("Model saved to {}".format(model_name + ".h5"))
print(history.history)
class MobileNetModel: #(ClassificationModel):
def __init__(self):
#super(CardModel, self).__init___(model_name='CardModel')
self.num_classes = 2
self.build_model()
return
def build_model(self):
# Initializing the model with random wights
self.arch = MobileNet(weights=None,
include_top=True,
classes=self.num_classes)
# Compiling model with optimization of RSM and cross entropy loss
self.arch.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
return
def train(self,
epochs,
train_images,
train_labels,
val_data,
batch_size,
initial_epoch=None):
history = self.arch.fit_generator(
train_generator,
epochs=epochs,
steps_per_epoch=train_generator.samples // batch_size,
validation_data=validation_generator,
validation_steps=validation_generator.samples // batch_size)
return history
def fit_gen(self, train_dir, val_dir, num_train, num_val, batch_size,
epochs):
gen = ImageDataGenerator(preprocessing_function=preprocess_input)
train_generator = gen.flow_from_directory(train_dir,
target_size=(224, 224),
batch_size=batch_size,
class_mode='categorical')
val_generator = gen.flow_from_directory(val_dir,
target_size=(224, 224),
batch_size=batch_size,
class_mode='categorical')
train_history = self.arch.fit_generator(
train_generator,
steps_per_epoch=(num_train // batch_size),
epochs=epochs,
validation_data=val_generator,
validation_steps=(num_val // batch_size))
return train_history
def save(self, output_dir):
model_path = os.path.join(output_dir, 'model.h5')
self.arch.save(model_path)
return
def load(self, input_path):
model_path = os.path.join(output_dir, 'model.h5')
self.arch = load_model(model_path)
return
def __repr__(self):
return str(self.arch.summary())
Plot metrics
"""
print(example.history.keys())
plt.plot(example.history['acc'])
plt.plot(example.history['val_acc'])
plt.xlabel('epochs')
plt.ylabel('accuracy')
plt.legend(['train', 'val'], loc = 'upper left')
plt.show()
path = '/content/drive/My Drive/Colab Notebooks/rmn1' #the path of the directory in which you have to save model
test_path = '/content/drive/My Drive/Colab Notebooks/Large_data/Test' #directory where test images are present
model.save(path + 'model.h5')
tflite_converter = tf.lite.TFLiteConverter.from_keras_model_file(path + 'model.h5')
tflite_model = tflite_converter.convert()
open(path + 'model_lite.tflite','wb').write(tflite_model)
with open('/content/drive/My Drive/Colab Notebooks/trainHistoryDict', 'wb') as f:
pickle.dump(example.history, f)
"""
Get the traffic sign from the gif/video.
"""
avoid_repeat_sign = 0;
final = []
x = GlobalAveragePooling2D()(x) #Add a GlobalAvgPooling
x = Dense(1024, activation='relu')(x)
out = Dense(NUM_CLASSES, activation='softmax')(x)
model = Model(inputs=model.input, outputs=out)
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
history = model.fit_generator(train_generator,
steps_per_epoch = train_generator.n // _batch,
epochs = _epochs,
verbose=1)
model.save(model_name)
plt.plot(history.history['loss'])
plt.plot(history.history['acc'])
plt.title('Model history')
plt.ylabel('Loss / Accuracy')
plt.xlabel('Epoch')
plt.legend(['Loss', 'Accuracy'], loc='upper left')
plt.savefig('accuracy_plot.png')
return model
mod = create_model(PATH,EPOCHS,32,False)
def load_testing_data(path,color,_shuffle):
x = blist([])
y = blist([])
main_path=os.listdir(path)
main_path.sort()
os.makedirs(onnx_dir, exist_ok=True)
with open(os.path.join(onnx_dir, 'model.onnx'), 'wb') as f:
f.write(model_proto.SerializeToString())
print('[INFO] DONE')
# --- Keras ---
print('[INFO] Keras MobileNet v1 downloading ...')
model = MobileNet(input_shape=None,
alpha=1.0,
depth_multiplier=1,
dropout=1e-3,
include_top=True,
weights='imagenet',
input_tensor=None,
pooling=None,
classes=1000)
save_dir = os.path.join(args.save_dir, 'keras/mobilenet_v1')
save_file = os.path.join(save_dir, 'mobilenet_v1.h5')
os.makedirs(save_dir, exist_ok=True)
model.save(save_file)
print('[INFO] DONE')
print('[INFO] Keras MobileNet v1 to ONNX ...')
save_dir = os.path.join(args.save_dir, 'keras/mobilenet_v1/onnx_model')
os.makedirs(save_dir, exist_ok=True)
save_file = os.path.join(save_dir, 'model.onnx')
onnx_model = keras2onnx.convert_keras(model, model.name)
onnx.save_model(onnx_model, save_file)
print('[INFO] DONE')
def train(dataset, architecture, task_name):
ROOT_MODELS = '/home/dembanakh/.ml-manager/tasks-weights/'
ROOT_DATASETS = '/home/dembanakh/.ml-manager/datasets/'
if dataset == 'IMAGENET':
if architecture == 'VGG16':
from keras.applications.vgg16 import VGG16
model = VGG16(weights='imagenet')
elif architecture == 'VGG19':
from keras.applications.vgg19 import VGG19
model = VGG19(weights='imagenet')
elif architecture == 'MobileNet':
from keras.applications.mobilenet import MobileNet
model = MobileNet(weights='imagenet')
elif architecture == 'ResNet':
from keras.applications.resnet import ResNet50, preprocess_input
model = ResNet50(weights='imagenet')
elif architecture == 'DenseNet':
from keras.applications.densenet import DenseNet121, preprocess_input
model = DenseNet121(weights='imagenet')
else:
return 0
model.compile(optimizer='adam',
metrics=['accuracy'],
loss='sparse_categorical_crossentropy')
model.save(ROOT_MODELS + task_name + '.h5')
else:
input_shape = (224, 224, 3)
batch_size = 1 # subject to change, but Azure server has little RAM
import os
import numpy as np
from keras.preprocessing import image
try:
samples = [i for i in os.listdir(dataset + '/samples')]
except OSError:
print 'There is no such directory', dataset + '/samples'
return 0
X = np.zeros((len(samples), input_shape[0], input_shape[1],
input_shape[2])) # maybe depends on architecture
y = np.zeros((len(samples), ))
if architecture == 'VGG16':
from keras.applications.vgg16 import VGG16, preprocess_input
model = VGG16()
for i in range(X.shape[0]):
X[i] = preprocess_input(X[i])
elif architecture == 'VGG19':
from keras.applications.vgg19 import VGG19, preprocess_input
model = VGG19()
for i in range(X.shape[0]):
X[i] = preprocess_input(X[i])
elif architecture == 'MobileNet':
from keras.applications.mobilenet import MobileNet, preprocess_input
model = MobileNet()
for i in range(X.shape[0]):
X[i] = preprocess_input(X[i])
elif architecture == 'ResNet':
from keras.applications.resnet import ResNet50, preprocess_input
model = ResNet50()
for i in range(X.shape[0]):
X[i] = preprocess_input(X[i])
elif architecture == 'DenseNet':
from keras.applications.densenet import DenseNet121, preprocess_input
model = DenseNet121()
for i in range(X.shape[0]):
X[i] = preprocess_input(X[i])
else:
return 0
for i, sample in enumerate(samples):
try:
img = image.load_img(dataset + '/samples/' + sample,
target_size=input_shape)
except IOError:
print 'Failed to open file', dataset + '/samples/' + sample
return 0
try:
f_lbl = open(
dataset + '/labels/' + sample.split('.')[0] + '.txt', 'r')
except IOError:
print 'Failed to open file', dataset + '/labels/' + sample.split(
'.')[0] + '.txt'
return 0
try:
y[i] = int(f_lbl.read())
except ValueError:
print 'File', dataset + '/labels/' + sample.split(
'.')[0] + '.txt', 'doesn\'t contain integer'
return 0
model.compile(optimizer='adam',
metrics=['accuracy'],
loss='sparse_categorical_crossentropy')
model.fit(X, y, batch_size=batch_size)
model.save(ROOT_MODELS + task_name + '.h5')
return 1
from keras.applications.mobilenet import MobileNet
INPUT_SHAPE = (128, 128, 3)
model = MobileNet(weights='imagenet', include_top=True, input_shape=INPUT_SHAPE)
model.save('model.h5')
