def convolutional(instruction=None,
read_mode=None,
preprocess=True,
data_path=None,
verbose=0,
new_folders=True,
image_column=None,
training_ratio=0.8,
fine_tune=False,
augmentation=True,
custom_arch=None,
pretrained=None,
epochs=10,
height=None,
width=None,
save_as_tfjs=None,
save_as_tflite=None,
generate_plots=True):
'''
Body of the convolutional function used that is called in the neural network query
if the data is presented in images.
:param many parameters: used to preprocess, tune, plot generation, and parameterizing the convolutional neural network trained.
:return dictionary that holds all the information for the finished model.
'''
# data_path = get_folder_dir()
logger("Generating datasets for classes")
LR = 0.001
plots = {}
if pretrained:
if not height:
height = 224
if not width:
width = 224
if height != 224 or width != 224:
raise ValueError(
"For pretrained models, both 'height' and 'width' must be 224."
)
if preprocess:
if custom_arch:
raise ValueError(
"If 'custom_arch' is not None, 'preprocess' must be set to false."
)
read_mode_info = set_distinguisher(data_path, read_mode)
read_mode = read_mode_info["read_mode"]
training_path = "/proc_training_set"
testing_path = "/proc_testing_set"
if read_mode == "setwise":
processInfo = setwise_preprocessing(data_path, new_folders, height,
width)
if not new_folders:
training_path = "/training_set"
testing_path = "/testing_set"
# if image dataset in form of csv
elif read_mode == "csvwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = csv_preprocessing(read_mode_info["csv_path"],
data_path, instruction,
image_column, training_ratio,
height, width)
# if image dataset in form of one folder containing class folders
elif read_mode == "classwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = classwise_preprocessing(data_path, training_ratio,
height, width)
else:
training_path = "/training_set"
testing_path = "/testing_set"
processInfo = already_processed(data_path)
num_channels = 3
color_mode = 'rgb'
if processInfo["gray_scale"]:
num_channels = 1
color_mode = 'grayscale'
input_shape = (processInfo["height"], processInfo["width"], num_channels)
input_single = (processInfo["height"], processInfo["width"])
num_classes = processInfo["num_categories"]
loss_func = ""
output_layer_activation = ""
if num_classes > 2:
loss_func = "categorical_crossentropy"
output_layer_activation = "softmax"
elif num_classes == 2:
num_classes = 1
loss_func = "binary_crossentropy"
output_layer_activation = "sigmoid"
logger("Creating convolutional neural network dynamically")
# Convolutional Neural Network
# Build model based on custom_arch configuration if given
if custom_arch:
with open(custom_arch, "r") as f:
custom_arch_dict = json.load(f)
custom_arch_json_string = json.dumps(custom_arch_dict)
model = model_from_json(custom_arch_json_string)
# Build an existing state-of-the-art model
elif pretrained:
arch_lower = pretrained.get('arch').lower()
# If user specifies value of pretrained['weights'] as 'imagenet', weights pretrained on ImageNet will be used
if 'weights' in pretrained and pretrained.get('weights') == 'imagenet':
# Load ImageNet pretrained weights
if arch_lower == "vggnet16":
base_model = VGG16(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "vggnet19":
base_model = VGG19(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet50":
base_model = ResNet50(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet101":
base_model = ResNet101(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet152":
base_model = ResNet152(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "mobilenet":
base_model = MobileNet(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = fine_tuned_model(base_model)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "mobilenetv2":
base_model = MobileNetV2(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = fine_tuned_model(base_model)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "densenet121":
base_model = DenseNet121(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = fine_tuned_model(base_model)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "densenet169":
base_model = DenseNet169(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = fine_tuned_model(base_model)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "densenet201":
base_model = DenseNet201(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = fine_tuned_model(base_model)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
else:
raise ModuleNotFoundError("arch \'" + pretrained.get('arch') +
"\' not supported.")
else:
# Randomly initialized weights
if arch_lower == "vggnet16":
model = VGG16(include_top=True,
weights=None,
classes=num_classes,
classifier_activation=output_layer_activation)
elif arch_lower == "vggnet19":
model = VGG19(include_top=True,
weights=None,
classes=num_classes,
classifier_activation=output_layer_activation)
elif arch_lower == "resnet50":
model = ResNet50(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "resnet101":
model = ResNet101(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "resnet152":
model = ResNet152(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "mobilenet":
model = MobileNet(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "mobilenetv2":
model = MobileNetV2(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "densenet121":
model = DenseNet121(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "densenet169":
model = DenseNet169(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "densenet201":
model = DenseNet201(include_top=True,
weights=None,
classes=num_classes)
else:
raise ModuleNotFoundError("arch \'" + pretrained.get('arch') +
"\' not supported.")
else:
model = Sequential()
# model.add(
# Conv2D(
# 64,
# kernel_size=3,
# activation="relu",
# input_shape=input_shape))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Conv2D(64, kernel_size=3, activation="relu"))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Flatten())
# model.add(Dense(num_classes, activation="softmax"))
# model.compile(
# optimizer="adam",
# loss=loss_func,
# metrics=['accuracy'])
model.add(
Conv2D(filters=64,
kernel_size=5,
activation="relu",
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=256, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(units=num_classes, activation="softmax"))
if pretrained and 'weights' in pretrained and pretrained.get(
'weights') == 'imagenet':
for layer in base_model.layers:
layer.trainable = False
opt = Adam(learning_rate=LR)
model.compile(optimizer=opt, loss=loss_func, metrics=['accuracy'])
logger("Located image data")
if augmentation:
train_data = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_data = ImageDataGenerator(rescale=1. / 255)
logger('Dataset augmented through zoom, shear, flip, and rescale')
else:
train_data = ImageDataGenerator()
test_data = ImageDataGenerator()
logger("->", "Optimal image size identified: {}".format(input_shape))
X_train = train_data.flow_from_directory(
data_path + training_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(16 if processInfo["train_size"] >= 16 else 1),
class_mode=loss_func[:loss_func.find("_")])
X_test = test_data.flow_from_directory(
data_path + testing_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(16 if processInfo["test_size"] >= 16 else 1),
class_mode=loss_func[:loss_func.find("_")])
if epochs <= 0:
raise BaseException("Number of epochs has to be greater than 0.")
print("\n")
logger('Training image model')
# model.summary()
history = model.fit_generator(
X_train,
steps_per_epoch=X_train.n // X_train.batch_size,
validation_data=X_test,
validation_steps=X_test.n // X_test.batch_size,
epochs=epochs,
verbose=verbose)
if fine_tune:
logger(
'->', 'Training accuracy: {}'.format(
history.history['accuracy'][len(history.history['accuracy']) -
1]))
logger(
'->',
'Validation accuracy: {}'.format(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
for layer in base_model.layers:
layer.trainable = True
opt = Adam(learning_rate=LR / 10)
model.compile(optimizer=opt, loss=loss_func, metrics=['accuracy'])
print("\n\n")
logger('Training fine tuned model')
fine_tuning_epoch = epochs + 10
history_fine = model.fit_generator(
X_train,
steps_per_epoch=X_train.n // X_train.batch_size,
validation_data=X_test,
validation_steps=X_test.n // X_test.batch_size,
epochs=fine_tuning_epoch,
initial_epoch=history.epoch[-1],
verbose=verbose)
#frozen model acc and loss history
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
#fine tuned model acc and loss history
acc += history_fine.history['accuracy']
val_acc += history_fine.history['val_accuracy']
loss += history_fine.history['loss']
val_loss += history_fine.history['val_loss']
if generate_plots:
plots = generate_fine_tuned_classification_plots(
acc, val_acc, loss, val_loss, epochs)
models = []
losses = []
accuracies = []
model_data = []
model_data.append(model)
models.append(history)
losses.append(
history.history["val_loss"][len(history.history["val_loss"]) - 1])
accuracies.append(
history.history['val_accuracy'][len(history.history['val_accuracy']) -
1])
# final_model = model_data[accuracies.index(max(accuracies))]
# final_hist = models[accuracies.index(max(accuracies))]
if generate_plots and not fine_tune:
plots = generate_classification_plots(models[len(models) - 1])
print("\n")
logger(
'->', 'Final training accuracy: {}'.format(
history.history['accuracy'][len(history.history['accuracy']) - 1]))
logger(
'->',
'Final validation accuracy: {}'.format(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
# storing values the model dictionary
number_of_examples = len(X_test.filenames)
number_of_generator_calls = math.ceil(number_of_examples /
(1.0 * X_test.batch_size))
test_labels = []
for i in range(0, int(number_of_generator_calls)):
test_labels.extend(np.array(X_test[i][1]))
predIdx = model.predict(X_test)
if output_layer_activation == "sigmoid":
real = [int(x) for x in test_labels]
ans = []
for i in range(len(predIdx)):
ans.append(int(round(predIdx[i][0])))
elif output_layer_activation == "softmax":
real = []
for ans in test_labels:
real.append(ans.argmax())
ans = []
for r in predIdx:
ans.append(r.argmax())
else:
print("NOT THE CASE")
logger("Stored model under 'convolutional_NN' key")
if save_as_tfjs:
tfjs.converters.save_keras_model(model, "tfjsmodel")
logger("Saved tfjs model under 'tfjsmodel' directory")
if save_as_tflite:
converter = tf.lite.TFLiteConverter.from_keras_model(model)
tflite_model = converter.convert()
open("model.tflite", "wb").write(tflite_model)
logger("Saved tflite model as 'model.tflite' ")
clearLog()
K.clear_session()
return {
'id': generate_id(),
'data_type': read_mode,
'data_path': data_path,
'data': {
'train': X_train,
'test': X_test
},
'shape': input_shape,
'res': {
'real': real,
'ans': ans
},
'model': model,
'plots': plots,
'losses': {
'training_loss': history.history['loss'],
'val_loss': history.history['val_loss']
},
'accuracy': {
'training_accuracy': history.history['accuracy'],
'validation_accuracy': history.history['val_accuracy']
},
'num_classes': (2 if num_classes == 1 else num_classes),
'data_sizes': {
'train_size': processInfo['train_size'],
'test_size': processInfo['test_size']
}
}
def regression_ann(instruction,
callback=False,
ca_threshold=None,
text=[],
dataset=None,
drop=None,
preprocess=True,
test_size=0.2,
random_state=49,
epochs=50,
generate_plots=True,
callback_mode='min',
maximizer="val_loss",
save_model=False,
save_path=os.getcwd(),
add_layer={}):
'''
Body of the regression function used that is called in the neural network query
if the data is numerical.
:param many parameters: used to preprocess, tune, plot generation, and parameterizing the neural network trained.
:return dictionary that holds all the information for the finished model.
'''
if dataset is None:
dataReader = DataReader(get_file())
else:
dataReader = DataReader(dataset)
logger("Reading in dataset")
data = dataReader.data_generator()
# data = pd.read_csv(self.dataset)
if drop is not None:
data.drop(drop, axis=1, inplace=True)
data, y, target, full_pipeline = initial_preprocessor(
data,
instruction,
preprocess,
ca_threshold,
text,
test_size=test_size,
random_state=random_state)
logger("->", "Target column found: {}".format(target))
X_train = data['train']
X_test = data['test']
# Target scaling
target_scaler = StandardScaler()
y_train = target_scaler.fit_transform(np.array(y['train']).reshape(-1, 1))
y_test = target_scaler.transform(np.array(y['test']).reshape(-1, 1))
logger("Establishing callback function")
models = []
losses = []
model_data = []
# callback function to store lowest loss value
es = EarlyStopping(monitor=maximizer,
mode=callback_mode,
verbose=0,
patience=5)
callback_value = None
if callback is not False:
callback_value = [es]
i = 0
# add_layer format: {<object> : list of indexs}
# get the first 3 layer model
model = get_keras_model_reg(data, i, add_layer)
logger("Training initial model")
history = model.fit(X_train,
y_train,
epochs=epochs,
validation_data=(X_test, y_test),
callbacks=callback_value,
verbose=0)
models.append(history)
model_data.append(model)
col_name = [[
"Initial number of layers ", "| Training Loss ", "| Test Loss "
]]
col_width = max(len(word) for row in col_name for word in row) + 2
for row in col_name:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
values = []
values.append(str(len(model.layers)))
values.append(
"| " +
str(history.history['loss'][len(history.history['val_loss']) - 1]))
values.append(
"| " +
str(history.history['val_loss'][len(history.history['val_loss']) - 1]))
datax = []
datax.append(values)
for row in datax:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
losses.append(history.history[maximizer][len(history.history[maximizer]) -
1])
# keeps running model and fit functions until the validation loss stops
# decreasing
logger("Testing number of layers")
col_name = [["Current number of layers", "| Training Loss", "| Test Loss"]]
col_width = max(len(word) for row in col_name for word in row) + 2
for row in col_name:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
datax = []
# while all(x > y for x, y in zip(losses, losses[1:])):
while (len(losses) <= 2
or losses[len(losses) - 1] < losses[len(losses) - 2]):
model = get_keras_model_reg(data, i, add_layer)
history = model.fit(X_train,
y_train,
callbacks=callback_value,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=0)
model_data.append(model)
models.append(history)
values = []
datax = []
values.append(str(len(model.layers)))
values.append(
"| " +
str(history.history['loss'][len(history.history['val_loss']) - 1]))
values.append("| " + str(history.history['val_loss'][
len(history.history['val_loss']) - 1]))
datax.append(values)
for row in datax:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
del values, datax
losses.append(
history.history[maximizer][len(history.history[maximizer]) - 1])
i += 1
# print((" " * 2 * counter)+ tabulate(datax, headers=col_name, tablefmt='orgtbl'))
final_model = model_data[losses.index(min(losses))]
final_hist = models[losses.index(min(losses))]
print("")
logger('->',
"Best number of layers found: " + str(len(final_model.layers)))
logger(
'->', "Training Loss: " +
str(final_hist.history['loss'][len(final_hist.history['val_loss']) -
1]))
logger(
'->', "Test Loss: " +
str(final_hist.history['val_loss'][len(final_hist.history['val_loss'])
- 1]))
# calls function to generate plots in plot generation
plots = {}
if generate_plots:
init_plots, plot_names = generate_regression_plots(
models[len(models) - 1], data, y)
for x in range(len(plot_names)):
plots[str(plot_names[x])] = init_plots[x]
if save_model:
save(final_model, save_model, save_path)
# stores values in the client object models dictionary field
print("")
logger("Stored model under 'regression_ANN' key")
clearLog()
K.clear_session()
return {
'id': generate_id(),
'model': final_model,
"target": target,
"num_classes": 1,
"plots": plots,
"preprocessor": full_pipeline,
"interpreter": target_scaler,
'test_data': {
'X': X_test,
'y': y_test
},
'losses': {
'training_loss': final_hist.history['loss'],
'val_loss': final_hist.history['val_loss']
}
}
def dcgan(instruction=None,
num_images=None,
preprocess=True,
data_path=None,
verbose=None,
epochs=None,
height=None,
width=None,
output_path=None):
#K.clear_session()
training_path = ""
logger("Preprocessing images")
num_channels = 3
if preprocess:
processInfo = single_class_preprocessing(data_path=data_path,
height=height,
width=width)
training_path = "proc_training_set"
num_channels = 1 if processInfo["gray_scale"] else 3
train_images = []
for file in os.listdir(data_path + "/" + training_path):
abs_path = os.path.join(data_path, training_path, file)
if os.path.isfile(abs_path):
train_images.append(cv2.imread(abs_path))
train_images = np.array(train_images)
logger("Building generator model and discriminator model")
### Build generator model and discriminator model
optimizer = Adam(0.0002, 0.5)
img_shape = (processInfo["height"], processInfo["width"], num_channels)
discriminator = build_discriminator(img_shape)
discriminator.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
generator = build_generator(img_shape)
generator.compile(loss='binary_crossentropy', optimizer=optimizer)
### Combine the generator and discriminators into one model ###
inp = Input(shape=100)
model_combined = Sequential()
model_combined.add(generator)
# Freeze discriminator's weights
discriminator.trainable = False
model_combined.add(discriminator)
model_combined.compile(loss='binary_crossentropy', optimizer=optimizer)
logger("Training Generative Adversarial Network")
loss_discriminator_history, acc_discriminator_history, loss_generator_history = train(
model_combined,
generator,
discriminator,
x_train=train_images,
epochs=epochs,
batch_size=32,
verbose=verbose)
logger("Generating output images")
generate_images(generator, num_images=num_images, output_path=data_path)
clearLog()
K.clear_session()
return {
'id': generate_id(),
'data': {
'train': train_images
},
'shape': (height, width, num_channels),
"model": model_combined,
'losses': {
'loss_discriminator_history': loss_discriminator_history,
'loss_generator_history': loss_generator_history
},
'accuracy': {
'acc_discriminator_history': acc_discriminator_history
}
}
def classification_ann(instruction,
callback=False,
dataset=None,
text=[],
ca_threshold=None,
preprocess=True,
callback_mode='min',
drop=None,
random_state=49,
test_size=0.2,
epochs=50,
generate_plots=True,
maximizer="val_accuracy",
save_model=False,
save_path=os.getcwd(),
add_layer={}):
'''
Body of the classification function used that is called in the neural network query
if the data is categorical.
:param many parameters: used to preprocess, tune, plot generation, and parameterizing the neural network trained.
:return dictionary that holds all the information for the finished model.
'''
if dataset is None:
dataReader = DataReader(get_file())
else:
dataReader = DataReader(dataset)
logger("Reading in dataset")
data = dataReader.data_generator()
if drop is not None:
data.drop(drop, axis=1, inplace=True)
data, y, remove, full_pipeline = initial_preprocessor(
data,
instruction,
preprocess,
ca_threshold,
text,
test_size=test_size,
random_state=random_state)
logger("->", "Target column found: {}".format(remove))
# Needed to make a custom label encoder due to train test split changes
# Can still be inverse transformed, just a bit of extra work
y = pd.concat([y['train'], y['test']], axis=0)
num_classes = len(np.unique(y))
if num_classes < 2:
raise Exception("Number of classes must be greater than or equal to 2")
X_train = data['train']
X_test = data['test']
if num_classes >= 2:
# ANN needs target one hot encoded for classification
one_hotencoder = OneHotEncoder()
y = pd.DataFrame(one_hotencoder.fit_transform(
np.reshape(y.values, (-1, 1))).toarray(),
columns=one_hotencoder.get_feature_names())
y_train = y.iloc[:len(X_train)]
y_test = y.iloc[len(X_train):]
models = []
losses = []
accuracies = []
model_data = []
logger("Establishing callback function")
# early stopping callback
es = EarlyStopping(monitor=maximizer, mode='max', verbose=0, patience=5)
callback_value = None
if callback is not False:
callback_value = [es]
i = 0
model = get_keras_model_class(data, i, num_classes, add_layer)
logger("Training initial model")
history = model.fit(X_train,
y_train,
callbacks=callback_value,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=0)
model_data.append(model)
models.append(history)
col_name = [[
"Initial number of layers ", "| Training Accuracy ", "| Test Accuracy "
]]
col_width = max(len(word) for row in col_name for word in row) + 2
for row in col_name:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
values = []
values.append(str(len(model.layers)))
values.append("| " + str(history.history['accuracy'][
len(history.history['val_accuracy']) - 1]))
values.append("| " + str(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
datax = []
datax.append(values)
for row in datax:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
# print((" " * 2 * counter)+ tabulate(datax, headers=col_name, tablefmt='orgtbl'))
losses.append(history.history[maximizer][len(history.history[maximizer]) -
1])
accuracies.append(
history.history['val_accuracy'][len(history.history['val_accuracy']) -
1])
# keeps running model and fit functions until the validation loss stops
# decreasing
logger("Testing number of layers")
col_name = [[
"Current number of layers", "| Training Accuracy", "| Test Accuracy"
]]
col_width = max(len(word) for row in col_name for word in row) + 2
for row in col_name:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
datax = []
# while all(x < y for x, y in zip(accuracies, accuracies[1:])):
while (len(accuracies) <= 2 or
accuracies[len(accuracies) - 1] > accuracies[len(accuracies) - 2]):
model = get_keras_model_class(data, i, num_classes, add_layer)
history = model.fit(X_train,
y_train,
callbacks=callback_value,
epochs=epochs,
validation_data=(X_test, y_test),
verbose=0)
values = []
datax = []
values.append(str(len(model.layers)))
values.append("| " + str(history.history['accuracy'][
len(history.history['accuracy']) - 1]))
values.append("| " + str(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
datax.append(values)
for row in datax:
print((" " * 2 * counter) + "| " +
("".join(word.ljust(col_width) for word in row)) + " |")
del values, datax
losses.append(
history.history[maximizer][len(history.history[maximizer]) - 1])
accuracies.append(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1])
models.append(history)
model_data.append(model)
i += 1
# print((" " * 2 * counter)+ tabulate(datax, headers=col_name, tablefmt='orgtbl'))
# del values, datax
final_model = model_data[accuracies.index(max(accuracies))]
final_hist = models[accuracies.index(max(accuracies))]
print("")
logger('->',
"Best number of layers found: " + str(len(final_model.layers)))
logger(
'->', "Training Accuracy: " + str(final_hist.history['accuracy'][
len(final_hist.history['val_accuracy']) - 1]))
logger(
'->', "Test Accuracy: " + str(final_hist.history['val_accuracy'][
len(final_hist.history['val_accuracy']) - 1]))
# genreates appropriate classification plots by feeding all information
plots = {}
if generate_plots:
plots = generate_classification_plots(models[len(models) - 1])
if save_model:
save(final_model, save_model, save_path)
print("")
logger("Stored model under 'classification_ANN' key")
clearLog()
K.clear_session()
# stores the values and plots into the object dictionary
return {
'id': generate_id(),
"model": final_model,
'num_classes': num_classes,
"plots": plots,
"target": remove,
"preprocessor": full_pipeline,
"interpreter": one_hotencoder,
'test_data': {
'X': X_test,
'y': y_test
},
'losses': {
'training_loss': final_hist.history['loss'],
'val_loss': final_hist.history['val_loss']
},
'accuracy': {
'training_accuracy': final_hist.history['accuracy'],
'validation_accuracy': final_hist.history['val_accuracy']
}
}
def train_xgboost(instruction,
dataset=None,
learning_rate=0.1,
n_estimators=1000,
ca_threshold=None,
max_depth=6,
min_child_weight=1,
gamma=0,
subsample=0.8,
colsample_bytree=0.8,
objective='binary:logistic',
random_state=27,
test_size=0.2,
text=[],
preprocess=True,
verbosity=0,
drop=None):
'''
function to train a xgboost algorithm
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
'''
logger("Reading in dataset")
dataReader = DataReader(dataset)
data = dataReader.data_generator()
if drop is not None:
data.drop(drop, axis=1, inplace=True)
logger("Preprocessing data")
data, y, target, full_pipeline = initial_preprocesser(
data,
instruction,
preprocess,
ca_threshold,
text,
test_size=test_size,
random_state=random_state)
logger("->", "Target column found: {}".format(target))
X_train = data['train']
y_train = y['train']
X_test = data['test']
y_test = y['test']
# classification_column = get_similar_column(getLabelwithInstruction(instruction), data)
num_classes = len(np.unique(y))
if num_classes > 2:
objective = 'multi:softmax'
# Needed to make a custom label encoder due to train test split changes
# Can still be inverse transformed, just a bit of extra work
y_vals = np.unique(pd.concat([y['train'], y['test']], axis=0))
label_mappings = sklearn.preprocessing.LabelEncoder()
label_mappings.fit(y_vals)
y_train = label_mappings.transform(y_train)
y_test = label_mappings.transform(y_test)
# Fitting to SVM and storing in the model dictionary
logger("Fitting XGBoost")
clf = XGBClassifier(learning_rate=learning_rate,
n_estimators=n_estimators,
max_depth=max_depth,
min_child_weight=min_child_weight,
gamma=gamma,
subsample=subsample,
colsample_bytree=colsample_bytree,
objective=objective,
verbosity=verbosity,
random_state=random_state)
clf.fit(X_train, y_train)
score = accuracy_score(clf.predict(X_test), y_test)
logger("->", "Accuracy found on testing set: {}".format(score))
logger('->', "Stored model under 'xgboost' key")
clearLog()
clearLog()
return {
'id': generate_id(),
"model": clf,
"target": target,
'num_classes': num_classes,
"accuracy": {
'cross_val_score': cross_val_score(
clf,
X_train,
y_train,
),
'accuracy_score': score
},
"accuracy_score": score,
"preprocesser": full_pipeline,
"interpreter": label_mappings,
'test_data': {
'X': X_test,
'y': y_test
}
}
def k_means_clustering(dataset=None,
scatters=[],
clusters=None,
preprocess=True,
generate_plots=True,
drop=None,
base_clusters=1,
verbose=0,
n_init=10,
max_iter=300,
random_state=42,
text=[]):
'''
function to train a k means clustering algorithm
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
'''
logger("Reading in dataset")
dataReader = DataReader(dataset)
data = dataReader.data_generator()
if drop is not None:
data.drop(drop, axis=1, inplace=True)
dataPandas = data.copy()
full_pipeline = None
if preprocess:
logger("Preprocessing data")
data, full_pipeline = clustering_preprocessor(data)
data = np.array(data)
modelStorage = []
inertiaStor = []
# processes dataset and runs KMeans algorithm on one cluster as
# baseline
if clusters is None:
i = base_clusters
logger("Creating unsupervised clustering task")
kmeans = KMeans(n_clusters=i,
random_state=random_state,
verbose=verbose,
n_init=n_init,
max_iter=max_iter).fit(data)
modelStorage.append(kmeans)
# stores SSE values in an array for later comparison
inertiaStor.append(kmeans.inertia_)
logger("Identifying best centroid count and optimizing accuracy")
col_name = [["Number of clusters ", "| Inertia "]]
col_width = max(len(word) for row in col_name for word in row) + 2
printtable(col_name, col_width)
values = []
values.append(str(i))
values.append("| " + str(inertiaStor[i - base_clusters]))
datax = []
datax.append(values)
printtable(datax, col_width)
i += 1
# continues to increase cluster size until SSE values don't decrease by
# 1000 - this value was decided based on precedence
while (all(earlier >= later
for earlier, later in zip(inertiaStor, inertiaStor[1:]))):
kmeans = KMeans(n_clusters=i,
random_state=random_state,
verbose=verbose,
n_init=n_init,
max_iter=max_iter).fit(data)
modelStorage.append(kmeans)
inertiaStor.append(kmeans.inertia_)
values = []
values.append(str(i))
values.append("| " + str(inertiaStor[i - base_clusters]))
datax = []
datax.append(values)
printtable(datax, col_width)
# minimize inertia up to 10000
i += 1
# checks to see if it should continue to run; need to improve this
# algorithm
if i > 3 and inertiaStor[len(inertiaStor) -
2] - 1000 <= inertiaStor[len(inertiaStor)
- 1]:
print()
break
# generates the clustering plots approiately
logger("->", "Optimal number of clusters found: {}".format(i))
logger("->",
"Final inertia of {}".format(inertiaStor[len(inertiaStor) - 1]))
else:
kmeans = KMeans(n_clusters=clusters,
random_state=random_state,
verbose=verbose,
n_init=n_init,
max_iter=max_iter).fit(data)
plots = {}
if generate_plots:
if clusters is None:
logger("Generating plots and storing in model")
init_plots, plot_names, elbow = generate_clustering_plots(
modelStorage[len(modelStorage) - 1], dataPandas, data,
scatters, inertiaStor, base_clusters)
for x in range(len(plot_names)):
plots[str(plot_names[x])] = init_plots[x]
plots['elbow'] = elbow
logger("Stored model under 'k_means_clustering' key")
clearLog()
# stores plots and information in the dictionary client model
return {
'id':
generate_id(),
"model":
(modelStorage[len(modelStorage) - 1] if clusters is None else kmeans),
"preprocesser":
full_pipeline,
"plots":
plots
}
def decision_tree(instruction,
dataset=None,
preprocess=True,
ca_threshold=None,
text=[],
test_size=0.2,
drop=None,
criterion='gini',
splitter='best',
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
ccp_alpha=0.0):
'''
function to train a decision tree algorithm.
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
'''
logger("Reading in dataset")
dataReader = DataReader(dataset)
data = dataReader.data_generator()
logger("Preprocessing data")
if drop is not None:
data.drop(drop, axis=1, inplace=True)
data, y, remove, full_pipeline = initial_preprocesser(
data, instruction, preprocess, ca_threshold, text)
logger("->", "Target column found: {}".format(remove))
X_train = data['train']
y_train = y['train']
X_test = data['test']
y_test = y['test']
# classification_column = get_similar_column(getLabelwithInstruction(instruction), data)
# Needed to make a custom label encoder due to train test split changes
# Can still be inverse transformed, just a bit of extra work
y_vals = np.unique(pd.concat([y['train'], y['test']], axis=0))
label_mappings = sklearn.preprocessing.LabelEncoder()
label_mappings.fit(y_vals)
y_train = label_mappings.transform(y_train)
y_test = label_mappings.transform(y_test)
logger("Labels being mapped to appropriate classes")
num_classes = len(np.unique(y))
# fitting and storing
logger("Fitting Decision Tree")
clf = tree.DecisionTreeClassifier(
criterion=criterion,
splitter=splitter,
max_depth=max_depth,
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_leaf_nodes=max_leaf_nodes,
min_impurity_decrease=min_impurity_decrease,
ccp_alpha=ccp_alpha)
clf = clf.fit(X_train, y_train)
score = accuracy_score(clf.predict(X_test), y_test)
logger("->", "Score found on testing set: {}".format(score))
logger("Stored model under 'decision_tree' key")
clearLog()
return {
'id': generate_id(),
"model": clf,
"target": remove,
'num_classes': num_classes,
"accuracy": {
'cross_val_score': cross_val_score(clf, X_train, y_train, cv=3),
'accuracy_score': score
},
"accuracy_score": score,
"preprocesser": full_pipeline,
"interpreter": label_mappings,
'test_data': {
'X': X_test,
'y': y_test
}
}
def nearest_neighbors(instruction=None,
dataset=None,
ca_threshold=None,
preprocess=True,
drop=None,
min_neighbors=3,
max_neighbors=10,
leaf_size=30,
p=2,
test_size=0.2,
random_state=49,
algorithm='auto',
text=[]):
'''
function to train a nearest neighbor algorithm
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
'''
logger("Reading in dataset")
# Reads in dataset
# data = pd.read_csv(self.dataset)
dataReader = DataReader(dataset)
data = dataReader.data_generator()
if drop is not None:
data.drop(drop, axis=1, inplace=True)
logger("Preprocessing data")
data, y, remove, full_pipeline = initial_preprocesser(
data,
instruction,
preprocess,
ca_threshold,
text,
test_size=test_size,
random_state=random_state)
logger("->", "Target column found: {}".format(remove))
X_train = data['train']
y_train = y['train']
X_test = data['test']
y_test = y['test']
# classification_column = get_similar_column(getLabelwithInstruction(instruction), data)
num_classes = len(np.unique(y))
# encodes the label dataset into 0's and 1's
y_vals = np.unique(pd.concat([y['train'], y['test']], axis=0))
label_mappings = sklearn.preprocessing.LabelEncoder()
label_mappings.fit(y_vals)
y_train = label_mappings.transform(y_train)
y_test = label_mappings.transform(y_test)
logger("Labels being mapped to appropriate classes")
models = []
scores = []
logger("Fitting nearest neighbors model")
logger("Identifying optimal number of neighbors")
# Tries all neighbor possibilities, based on either defaults or user
# specified values
num_neighbors = []
for x in range(min_neighbors, max_neighbors):
knn = KNeighborsClassifier(n_neighbors=x,
leaf_size=leaf_size,
p=p,
algorithm=algorithm)
knn.fit(X_train, y_train)
models.append(knn)
scores.append(accuracy_score(knn.predict(X_test), y_test))
num_neighbors.append(x)
logger(
"->", "Optimal number of neighbors found: {}".format(
num_neighbors[scores.index(max(scores))]))
logger(
"->", "Accuracy found on testing set: {}".format(scores[scores.index(
max(scores))]))
logger("Stored model under 'nearest_neighbors' key")
knn = models[scores.index(min(scores))]
clearLog()
return {
'id': generate_id(),
"model": knn,
'num_classes': num_classes,
"accuracy": {
'accuracy_score': scores[scores.index(max(scores))],
'cross_val_score': cross_val_score(knn, X_train, y_train, cv=3)
},
"preprocesser": full_pipeline,
"interpreter": label_mappings,
'test_data': {
'X': X_test,
'y': y_test
},
"target": remove
}
clearLog()
def train_svm(instruction,
dataset=None,
test_size=0.2,
kernel='linear',
text=[],
preprocess=True,
ca_threshold=None,
drop=None,
cross_val_size=0.3,
degree=3,
gamma='scale',
coef0=0.0,
max_iter=-1,
random_state=49):
'''
function to train a support vector machine clustering algorithm
:param many params: used to hyperparametrize the function.
:return a dictionary object with all of the information for the algorithm.
'''
logger("Reading in dataset")
dataReader = DataReader(dataset)
data = dataReader.data_generator()
if drop is not None:
data.drop(drop, axis=1, inplace=True)
logger("Preprocessing data")
data, y, target, full_pipeline = initial_preprocesser(
data,
instruction,
preprocess,
ca_threshold,
text,
test_size=test_size,
random_state=random_state)
logger("->", "Target column found: {}".format(target))
X_train = data['train']
y_train = y['train']
X_test = data['test']
y_test = y['test']
# classification_column = get_similar_column(getLabelwithInstruction(instruction), data)
num_classes = len(np.unique(y))
# Needed to make a custom label encoder due to train test split changes
# Can still be inverse transformed, just a bit of extra work
y_vals = np.unique(pd.concat([y['train'], y['test']], axis=0))
label_mappings = sklearn.preprocessing.LabelEncoder()
label_mappings.fit(y_vals)
y_train = label_mappings.transform(y_train)
y_test = label_mappings.transform(y_test)
# Fitting to SVM and storing in the model dictionary
logger("Fitting Support Vector Machine")
clf = svm.SVC(kernel=kernel,
degree=degree,
gamma=gamma,
coef0=coef0,
max_iter=max_iter)
clf.fit(X_train, y_train)
score = accuracy_score(clf.predict(X_test), y_test)
logger("->", "Accuracy found on testing set: {}".format(score))
logger('->', "Stored model under 'svm' key")
clearLog()
return {
'id': generate_id(),
"model": clf,
'num_classes': num_classes,
"accuracy": {
'cross_val_score': cross_val_score(clf, X_train, y_train),
'accuracy_score': score
},
"target": target,
"preprocesser": full_pipeline,
"interpreter": label_mappings,
'test_data': {
'X': X_test,
'y': y_test
}
}
clearLog()
def convolutional(instruction=None,
read_mode=None,
preprocess=True,
verbose=0,
data_path=os.getcwd(),
new_folders=True,
image_column=None,
training_ratio=0.8,
augmentation=True,
epochs=10,
height=None,
width=None):
'''
Body of the convolutional function used that is called in the neural network query
if the data is presented in images.
:param many parameters: used to preprocess, tune, plot generation, and parameterizing the convolutional neural network trained.
:return dictionary that holds all the information for the finished model.
'''
logger("Generating datasets for classes")
if preprocess:
read_mode_info = set_distinguisher(data_path, read_mode)
read_mode = read_mode_info["read_mode"]
training_path = "/proc_training_set"
testing_path = "/proc_testing_set"
if read_mode == "setwise":
processInfo = setwise_preprocessing(data_path, new_folders, height,
width)
if not new_folders:
training_path = "/training_set"
testing_path = "/testing_set"
# if image dataset in form of csv
elif read_mode == "csvwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = csv_preprocessing(read_mode_info["csv_path"],
data_path, instruction,
image_column, training_ratio,
height, width)
# if image dataset in form of one folder containing class folders
elif read_mode == "classwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = classwise_preprocessing(data_path, training_ratio,
height, width)
else:
training_path = "/training_set"
testing_path = "/testing_set"
processInfo = already_processed(data_path)
num_channels = 3
color_mode = 'rgb'
if processInfo["gray_scale"]:
num_channels = 1
color_mode = 'grayscale'
input_shape = (processInfo["height"], processInfo["width"], num_channels)
input_single = (processInfo["height"], processInfo["width"])
num_classes = processInfo["num_categories"]
loss_func = ""
if num_classes > 2:
loss_func = "categorical_crossentropy"
elif num_classes == 2:
loss_func = "binary_crossentropy"
logger("Creating convolutional neural network dynamically")
# Convolutional Neural Network
model = Sequential()
# model.add(
# Conv2D(
# 64,
# kernel_size=3,
# activation="relu",
# input_shape=input_shape))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Conv2D(64, kernel_size=3, activation="relu"))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Flatten())
# model.add(Dense(num_classes, activation="softmax"))
# model.compile(
# optimizer="adam",
# loss=loss_func,
# metrics=['accuracy'])
model.add(
Conv2D(filters=64,
kernel_size=5,
activation="relu",
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=256, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(units=num_classes, activation="softmax"))
model.compile(optimizer="adam", loss=loss_func, metrics=['accuracy'])
logger("Located image data")
if augmentation:
train_data = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_data = ImageDataGenerator(rescale=1. / 255)
logger('Dataset augmented through zoom, shear, flip, and rescale')
else:
train_data = ImageDataGenerator()
test_data = ImageDataGenerator()
logger("->", "Optimal image size identified: {}".format(input_shape))
X_train = train_data.flow_from_directory(
data_path + training_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(32 if processInfo["train_size"] >= 32 else 1),
class_mode=loss_func[:loss_func.find("_")])
X_test = test_data.flow_from_directory(
data_path + testing_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(32 if processInfo["test_size"] >= 32 else 1),
class_mode=loss_func[:loss_func.find("_")])
if epochs < 0:
raise BaseException("Number of epochs has to be greater than 0.")
logger('Training image model')
history = model.fit_generator(
X_train,
steps_per_epoch=X_train.n // X_train.batch_size,
validation_data=X_test,
validation_steps=X_test.n // X_test.batch_size,
epochs=epochs,
verbose=verbose)
logger(
'->', 'Final training accuracy: {}'.format(
history.history['accuracy'][len(history.history['accuracy']) - 1]))
logger(
'->',
'Final validation accuracy: {}'.format(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
# storing values the model dictionary
logger("Stored model under 'convolutional_NN' key")
clearLog()
return {
'id': generate_id(),
'data_type': read_mode,
'data_path': data_path,
'data': {
'train': X_train,
'test': X_test
},
'shape': input_shape,
"model": model,
'losses': {
'training_loss': history.history['loss'],
'val_loss': history.history['val_loss']
},
'accuracy': {
'training_accuracy': history.history['accuracy'],
'validation_accuracy': history.history['val_accuracy']
},
'num_classes': (2 if num_classes == 1 else num_classes),
'data_sizes': {
'train_size': processInfo['train_size'],
'test_size': processInfo['test_size']
}
}
def convolutional(instruction=None,
read_mode=None,
preprocess=True,
data_path=None,
verbose=0,
new_folders=True,
image_column=None,
training_ratio=0.8,
augmentation=True,
custom_arch=None,
pretrained=None,
epochs=10,
height=None,
width=None):
'''
Body of the convolutional function used that is called in the neural network query
if the data is presented in images.
:param many parameters: used to preprocess, tune, plot generation, and parameterizing the convolutional neural network trained.
:return dictionary that holds all the information for the finished model.
'''
# data_path = get_folder_dir()
logger("Generating datasets for classes")
if pretrained:
if not height:
height = 224
if not width:
width = 224
if height != 224 or width != 224:
raise ValueError(
"For pretrained models, both 'height' and 'width' must be 224."
)
if preprocess:
if custom_arch:
raise ValueError(
"If 'custom_arch' is not None, 'preprocess' must be set to false."
)
read_mode_info = set_distinguisher(data_path, read_mode)
read_mode = read_mode_info["read_mode"]
training_path = "/proc_training_set"
testing_path = "/proc_testing_set"
if read_mode == "setwise":
processInfo = setwise_preprocessing(data_path, new_folders, height,
width)
if not new_folders:
training_path = "/training_set"
testing_path = "/testing_set"
# if image dataset in form of csv
elif read_mode == "csvwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = csv_preprocessing(read_mode_info["csv_path"],
data_path, instruction,
image_column, training_ratio,
height, width)
# if image dataset in form of one folder containing class folders
elif read_mode == "classwise":
if training_ratio <= 0 or training_ratio >= 1:
raise BaseException(f"Test ratio must be between 0 and 1.")
processInfo = classwise_preprocessing(data_path, training_ratio,
height, width)
else:
training_path = "/training_set"
testing_path = "/testing_set"
processInfo = already_processed(data_path)
num_channels = 3
color_mode = 'rgb'
if processInfo["gray_scale"]:
num_channels = 1
color_mode = 'grayscale'
input_shape = (processInfo["height"], processInfo["width"], num_channels)
input_single = (processInfo["height"], processInfo["width"])
num_classes = processInfo["num_categories"]
loss_func = ""
output_layer_activation = ""
if num_classes > 2:
loss_func = "categorical_crossentropy"
output_layer_activation = "softmax"
elif num_classes == 2:
num_classes = 1
loss_func = "binary_crossentropy"
output_layer_activation = "sigmoid"
logger("Creating convolutional neural netwwork dynamically")
# Convolutional Neural Network
# Build model based on custom_arch configuration if given
if custom_arch:
with open(custom_arch, "r") as f:
custom_arch_dict = json.load(f)
custom_arch_json_string = json.dumps(custom_arch_dict)
model = model_from_json(custom_arch_json_string)
# Build an existing state-of-the-art model
elif pretrained:
arch_lower = pretrained.get('arch').lower()
# If user specifies value of pretrained['weights'] as 'imagenet', weights pretrained on ImageNet will be used
if 'weights' in pretrained and pretrained.get('weights') == 'imagenet':
# Load ImageNet pretrained weights
if arch_lower == "vggnet16":
base_model = VGG16(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "vggnet19":
base_model = VGG19(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
x = Dense(4096)(x)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet50":
base_model = ResNet50(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = Flatten()(base_model.output)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet101":
base_model = ResNet101(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
elif arch_lower == "resnet152":
base_model = ResNet152(include_top=False,
weights='imagenet',
input_shape=input_shape)
x = GlobalAveragePooling2D()(base_model.output)
x = Dropout(0.5)(x)
pred = Dense(num_classes,
activation=output_layer_activation)(x)
model = Model(base_model.input, pred)
else:
raise ModuleNotFoundError("arch \'" + pretrained.get('arch') +
"\' not supported.")
else:
# Randomly initialized weights
if arch_lower == "vggnet16":
model = VGG16(include_top=True,
weights=None,
classes=num_classes,
classifier_activation=output_layer_activation)
elif arch_lower == "vggnet19":
model = VGG19(include_top=True,
weights=None,
classes=num_classes,
classifier_activation=output_layer_activation)
elif arch_lower == "resnet50":
model = ResNet50(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "resnet101":
model = ResNet101(include_top=True,
weights=None,
classes=num_classes)
elif arch_lower == "resnet152":
model = ResNet152(include_top=True,
weights=None,
classes=num_classes)
else:
raise ModuleNotFoundError("arch \'" + pretrained.get('arch') +
"\' not supported.")
else:
model = Sequential()
# model.add(
# Conv2D(
# 64,
# kernel_size=3,
# activation="relu",
# input_shape=input_shape))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Conv2D(64, kernel_size=3, activation="relu"))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Flatten())
# model.add(Dense(num_classes, activation="softmax"))
# model.compile(
# optimizer="adam",
# loss=loss_func,
# metrics=['accuracy'])
model.add(
Conv2D(filters=64,
kernel_size=5,
activation="relu",
input_shape=input_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Conv2D(filters=64, kernel_size=3, activation="relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(units=256, activation="relu"))
model.add(Dropout(0.25))
model.add(Dense(units=num_classes, activation="softmax"))
model.compile(optimizer="adam", loss=loss_func, metrics=['accuracy'])
logger("Located image data")
if augmentation:
train_data = ImageDataGenerator(rescale=1. / 255,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_data = ImageDataGenerator(rescale=1. / 255)
logger('Dataset augmented through zoom, shear, flip, and rescale')
else:
train_data = ImageDataGenerator()
test_data = ImageDataGenerator()
logger("->", "Optimal image size identified: {}".format(input_shape))
X_train = train_data.flow_from_directory(
data_path + training_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(16 if processInfo["train_size"] >= 16 else 1),
class_mode=loss_func[:loss_func.find("_")])
X_test = test_data.flow_from_directory(
data_path + testing_path,
target_size=input_single,
color_mode=color_mode,
batch_size=(16 if processInfo["test_size"] >= 16 else 1),
class_mode=loss_func[:loss_func.find("_")])
if epochs <= 0:
raise BaseException("Number of epochs has to be greater than 0.")
logger('Training image model')
history = model.fit_generator(
X_train,
steps_per_epoch=X_train.n // X_train.batch_size,
validation_data=X_test,
validation_steps=X_test.n // X_test.batch_size,
epochs=epochs,
verbose=verbose)
logger(
'->', 'Final training accuracy: {}'.format(
history.history['accuracy'][len(history.history['accuracy']) - 1]))
logger(
'->',
'Final validation accuracy: {}'.format(history.history['val_accuracy'][
len(history.history['val_accuracy']) - 1]))
# storing values the model dictionary
logger("Stored model under 'convolutional_NN' key")
clearLog()
return {
'id': generate_id(),
'data_type': read_mode,
'data_path': data_path,
'data': {
'train': X_train,
'test': X_test
},
'shape': input_shape,
"model": model,
'losses': {
'training_loss': history.history['loss'],
'val_loss': history.history['val_loss']
},
'accuracy': {
'training_accuracy': history.history['accuracy'],
'validation_accuracy': history.history['val_accuracy']
},
'num_classes': (2 if num_classes == 1 else num_classes),
'data_sizes': {
'train_size': processInfo['train_size'],
'test_size': processInfo['test_size']
}
}