def run(_run, image_shape, train_info, data_dir, train_pairs, valid_pairs,
train_shuffle, valid_shuffle, subdirectories, architecture, weights,
batch_size, last_base_layer, pooling, device, opt_params, dropout_rate,
resuming_ckpt, ckpt, steps_per_epoch, epochs, validation_steps,
workers, use_multiprocessing, initial_epoch, early_stop_patience,
use_gram_matrix, dense_layers, limb_weights, trainable_limbs,
outputs_meta):
report_dir = _run.observers[0].dir
print('reading train-info...')
outputs, name_map = load_multiple_outputs(train_info,
outputs_meta,
encode='sparse')
g = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
zoom_range=.2,
rotation_range=.2,
height_shift_range=.2,
width_shift_range=.2,
fill_mode='reflect',
preprocessing_function=get_preprocess_fn(architecture))
print('loading train meta-data...')
train_data = BalancedDirectoryPairsMultipleOutputsSequence(
os.path.join(data_dir, 'train'),
outputs,
name_map,
g,
batch_size=batch_size,
target_size=image_shape[:2],
subdirectories=subdirectories,
shuffle=train_shuffle,
pairs=train_pairs)
print('loading valid meta-data...')
valid_data = BalancedDirectoryPairsMultipleOutputsSequence(
os.path.join(data_dir, 'valid'),
outputs,
name_map,
g,
batch_size=batch_size,
target_size=image_shape[:2],
subdirectories=subdirectories,
shuffle=valid_shuffle,
pairs=valid_pairs)
with tf.device(device):
print('building...')
model = build_siamese_model(
image_shape,
architecture,
dropout_rate,
weights,
last_base_layer=last_base_layer,
use_gram_matrix=use_gram_matrix,
dense_layers=dense_layers,
pooling=pooling,
include_base_top=False,
include_top=True,
trainable_limbs=trainable_limbs,
limb_weights=limb_weights,
predictions_activation=[o['a'] for o in outputs_meta],
predictions_name=[o['n'] for o in outputs_meta],
classes=[o['u'] for o in outputs_meta],
embedding_units=[o['e'] for o in outputs_meta],
joints=[o['j'] for o in outputs_meta])
print('siamese model summary:')
model.summary()
if resuming_ckpt:
print('loading weights...')
model.load_weights(resuming_ckpt)
model.compile(optimizer=optimizers.Adam(**opt_params),
loss=dict((o['n'] + '_binary_predictions', o['l'])
for o in outputs_meta),
metrics=dict((o['n'] + '_binary_predictions', o['m'])
for o in outputs_meta))
print('training from epoch %i...' % initial_epoch)
try:
model.fit_generator(
train_data,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_data=valid_data,
validation_steps=validation_steps,
initial_epoch=initial_epoch,
use_multiprocessing=use_multiprocessing,
workers=workers,
verbose=2,
callbacks=[
callbacks.TerminateOnNaN(),
callbacks.EarlyStopping(patience=early_stop_patience),
callbacks.ReduceLROnPlateau(min_lr=1e-10,
patience=early_stop_patience //
3),
callbacks.TensorBoard(report_dir, batch_size=batch_size),
callbacks.ModelCheckpoint(os.path.join(report_dir, ckpt),
save_best_only=True,
verbose=1),
])
except KeyboardInterrupt:
print('interrupted by user')
else:
print('done')
def train (dataset, model, le, plot='plot.png'):
"""
Function Wrapper to initialize training using Keras Model.
"""
args = {
'dataset': dataset,
'model': model,
'le': le,
'plot': plot
}
# initialize the initial learning rate, batch size, and number of
# epochs to train for
INIT_LR = 1e-4
BS = 8
EPOCHS = 50
# grab the list of images in our dataset directory, then initialize
# the list of data (i.e., images) and class images
print("[INFO] loading images...")
imagePaths = list(paths.list_images(args["dataset"]))
data = []
labels = []
for imagePath in imagePaths:
# extract the class label from the filename, load the image and
# resize it to be a fixed 32x32 pixels, ignoring aspect ratio
label = imagePath.split(os.path.sep)[-2]
try:
image = cv2.imread(imagePath)
image = cv2.resize(image, (32, 32))
except Exception as e:
print ("Image %s is broken. Continue.." % imagePath.split(os.path.sep)[-2:])
continue
# update the data and labels lists, respectively
data.append(image)
labels.append(label)
# convert the data into a NumPy array, then preprocess it by scaling
# all pixel intensities to the range [0, 1]
data = np.array(data, dtype="float") / 255.0
# encode the labels (which are currently strings) as integers and then
# one-hot encode them
le = LabelEncoder()
labels = le.fit_transform(labels)
labels = np_utils.to_categorical(labels, 2)
# partition the data into training and testing splits using 75% of
# the data for training and the remaining 25% for testing
(trainX, testX, trainY, testY) = train_test_split(data, labels,
test_size=0.25, random_state=42)
# construct the training image generator for data augmentation
aug = ImageDataGenerator(rotation_range=20,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
# vertical_flip=True,
fill_mode="nearest"
)
# initialize the optimizer and model
# print("[INFO] compiling model...")
# opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS)
# model_builder = Model(width=32, height=32, depth=3, classes=len(le.classes_))
# model = model_builder.build_liveness()
# VGG optimizer and model
print("[INFO] compiling model...")
opt = Adam(lr=INIT_LR, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6)
model_builder = Model(width=32, height=32, depth=3, classes=len(le.classes_))
model = model_builder.build_VGG()
nFreeze = 10
# freeze layers
for layer in model.layers[:nFreeze]:
layer.trainable = False
# model compilation
model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"])
# train the network
print("[INFO] training network for {} epochs...".format(EPOCHS))
earlyStopping = callbacks.EarlyStopping(monitor='val_acc',
patience=10,
verbose=0, mode='auto')
H = model.fit_generator(
aug.flow(trainX, trainY, batch_size=BS),
callbacks=[earlyStopping],
validation_data=(testX, testY),
steps_per_epoch=len(trainX) // BS,
epochs=EPOCHS
)
# evaluate the network
print("[INFO] evaluating network...")
predictions = model.predict(testX, batch_size=BS)
print(classification_report(testY.argmax(axis=1),
predictions.argmax(axis=1), target_names=le.classes_))
# save the network to disk
print("[INFO] serializing network to '{}'...".format(args["model"]))
modelpath = 'models/' + args['model']
model.save(modelpath)
# save the label encoder to disk
f = open(args["le"], "wb")
f.write(pickle.dumps(le))
f.close()
# plot the training loss and accuracy
plt.style.use("ggplot")
plt.figure()
print(H.history.keys())
plt.plot(np.arange(0, EPOCHS), H.history["loss"], label="train_loss")
plt.plot(np.arange(0, EPOCHS), H.history["val_loss"], label="val_loss")
plt.plot(np.arange(0, EPOCHS), H.history["accuracy"], label="train_acc")
plt.plot(np.arange(0, EPOCHS), H.history["val_accuracy"], label="val_acc")
plt.title("Training Loss and Accuracy on Dataset")
plt.xlabel("Epoch #")
plt.ylabel("Loss/Accuracy")
plt.legend(loc="lower left")
plotpath = 'plots/' + args["plot"]
plt.savefig(plotpath)
plt.show()
def main():
def load_scan2(path):
print("load_scan2")
list1 = []
for dirName, subdirList, fileList in os.walk(path):
for filename in fileList:
if ".dcm" in filename.lower():
list1.append(os.path.join(dirName, filename))
return list1
bc = []
bm = []
mm = []
mc = []
img = []
final = []
final_1 = []
a = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/benign-calc'
)
for i in range(0, 196):
# bc.append(pdicom.read_file(a[i]))
final.append(pdicom.read_file(a[i]))
bclen = len(a)
a = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/benign-calc'
)
for i in range(0, 196):
# bc.append(pdicom.read_file(a[i]))
final.append(pdicom.read_file(a[i]))
bclen = bclen + len(a)
b = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/benign-mass'
)
for i in range(0, 196):
# bm.append(pdicom.read_file(b[i]))
final.append(pdicom.read_file(b[i]))
bmlen = len(b)
b = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/benign-mass'
)
for i in range(0, 196):
# bm.append(pdicom.read_file(b[i]))
final.append(pdicom.read_file(b[i]))
bmlen = bmlen + len(b)
c = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/malignant-mass'
)
for i in range(0, 196):
# mm.append(pdicom.read_file(c[i]))
final.append(pdicom.read_file(c[i]))
mmlen = len(c)
# c = load_scan2('/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/malignant-mass')
for i in range(0, 196):
# mm.append(pdicom.read_file(c[i]))
final.append(pdicom.read_file(c[i]))
mmlen = mmlen + len(c)
e = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/malignant-calc'
)
for i in range(0, 196):
# mc.append(pdicom.read_file(e[i]))
final.append(pdicom.read_file(e[i]))
mclen = len(e)
e = load_scan2(
'/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/malignant-calc'
)
for i in range(0, 196):
# mc.append(pdicom.read_file(e[i]))
final.append(pdicom.read_file(e[i]))
mclen = mclen + len(e)
print(len(final))
for i in range(0, len(final)):
img.append(final[i].pixel_array)
for i in range(0, 500):
img[i] = img[i].astype(float)
final_1.append(np.stack((img[i], ) * 3, axis=-1))
for i in range(500, 1000):
img[i] = img[i].astype(float)
final_1.append(np.stack((img[i], ) * 3, axis=-1))
for i in range(1000, 1568):
img[i] = img[i].astype(float)
final_1.append(np.stack((img[i], ) * 3, axis=-1))
final_1 = np.array(final_1)
print(final_1[0].shape)
num_classes = 4
num_of_samples = len(final_1)
labels = np.ones((num_of_samples, ), dtype='int64')
labels[0:392] = 0
labels[392:784] = 1
labels[784:1176] = 2
labels[1176:1568] = 3
print("labels")
L = np_utils.to_categorical(labels, num_classes)
final_1, L = shuffle(final_1, L, random_state=2)
print("shuffle")
train_x, val_x, train_y, val_y = train_test_split(final_1,
L,
stratify=L,
test_size=0.2)
print("split")
print(len(train_x), len(train_y), len(val_x), len(val_y))
train_y = np.array(train_y)
print("train_y")
train_x = np.array(train_x)
print("train_x")
val_x = np.array(val_x)
val_y = np.array(val_y)
print("model")
for i in range(0, len(train_x)):
train_x[i] = train_x[i] / 255
print("normalize")
model = resnet_pseudo()
model.compile(loss="categorical_crossentropy",
optimizer="adagrad",
metrics=["accuracy"])
filename = 'model_train_new2.csv'
csv_log = callbacks.CSVLogger(filename, separator=',', append=False)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=0,
verbose=0,
mode='min')
filepath = "/home/genomics/PycharmProjects/BreastCancerPredictor/try10.hdf5"
checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
callbacks_list = [csv_log, early_stopping, checkpoint]
hist = model.fit(train_x,
train_y,
batch_size=10,
epochs=4,
verbose=1,
validation_data=(val_x, val_y),
callbacks=callbacks_list)
model.save('model10.hdf5')
def calibrate(target, source, sourceIndex, predLabel, path):
mmdNetLayerSizes = [25, 25]
l2_penalty = 1e-2
init = lambda shape, name: initializations.normal(
shape, scale=.1e-4, name=name)
space_dim = target.X.shape[1]
calibInput = Input(shape=(space_dim, ))
block1_bn1 = BatchNormalization()(calibInput)
block1_a1 = Activation('relu')(block1_bn1)
block1_w1 = Dense(mmdNetLayerSizes[0],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a1)
block1_bn2 = BatchNormalization()(block1_w1)
block1_a2 = Activation('relu')(block1_bn2)
block1_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block1_a2)
block1_output = merge([block1_w2, calibInput], mode='sum')
block2_bn1 = BatchNormalization()(block1_output)
block2_a1 = Activation('relu')(block2_bn1)
block2_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a1)
block2_bn2 = BatchNormalization()(block2_w1)
block2_a2 = Activation('relu')(block2_bn2)
block2_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block2_a2)
block2_output = merge([block2_w2, block1_output], mode='sum')
block3_bn1 = BatchNormalization()(block2_output)
block3_a1 = Activation('relu')(block3_bn1)
block3_w1 = Dense(mmdNetLayerSizes[1],
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a1)
block3_bn2 = BatchNormalization()(block3_w1)
block3_a2 = Activation('relu')(block3_bn2)
block3_w2 = Dense(space_dim,
activation='linear',
W_regularizer=l2(l2_penalty),
init=init)(block3_a2)
block3_output = merge([block3_w2, block2_output], mode='sum')
calibMMDNet = Model(input=calibInput, output=block3_output)
n = target.X.shape[0]
p = np.random.permutation(n)
toTake = p[range(int(.2 * n))]
targetXMMD = target.X[toTake]
targetYMMD = target.y[toTake]
targetXMMD = targetXMMD[targetYMMD != 0]
targetYMMD = targetYMMD[targetYMMD != 0]
targetYMMD = np.reshape(targetYMMD, (-1, 1))
n = source.X.shape[0]
p = np.random.permutation(n)
toTake = p[range(int(.2 * n))]
sourceXMMD = source.X[toTake]
sourceYMMD = predLabel[toTake]
sourceXMMD = sourceXMMD[sourceYMMD != 0]
sourceYMMD = sourceYMMD[sourceYMMD != 0]
sourceYMMD = np.reshape(sourceYMMD, (-1, 1))
lrate = LearningRateScheduler(step_decay)
optimizer = opt.rmsprop(lr=0.0)
calibMMDNet.compile(
optimizer=optimizer,
loss=lambda y_true, y_pred: cf.MMD(
block3_output, targetXMMD, MMDTargetValidation_split=0.1).
KerasCost(y_true, y_pred))
sourceLabels = np.zeros(sourceXMMD.shape[0])
calibMMDNet.fit(sourceXMMD,
sourceLabels,
nb_epoch=500,
batch_size=1000,
validation_split=0.1,
verbose=0,
callbacks=[
lrate,
mn.monitorMMD(sourceXMMD, sourceYMMD, targetXMMD,
targetYMMD, calibMMDNet.predict),
cb.EarlyStopping(monitor='val_loss',
patience=20,
mode='auto')
])
plt.close('all')
calibMMDNet.save_weights(
os.path.join(io.DeepLearningRoot(),
path + '/ResNet' + str(sourceIndex) + '.h5'))
calibrateSource = Sample(calibMMDNet.predict(source.X), source.y)
calibMMDNet = None
return calibrateSource
# model.add(PReLU())
# model.add(BatchNormalization())
# model.add(Dropout(0.2))
# model.add(Dense(3, init='he_normal', activation='softmax'))
# adam = optimizers.Adam(lr=1.0)
# model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# return(model)
# y_arr = pd.DataFrame.as_matrix(pd.DataFrame(y)).ravel()
# y_arr = np.array(y)
# fix random seed for reproducibility
seed = 7
earlyStopping = callbacks.EarlyStopping(monitor='loss',
patience=1,
verbose=0,
mode='auto')
model = KerasClassifier(build_fn=create_model,
epochs=40,
batch_size=50,
verbose=1)
# evaluate using 10-fold cross validation
kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)
results = model_selection.cross_val_score(
model,
x_train,
encoded_Y,
cv=kfold,
scoring='neg_log_loss',
fit_params={'callbacks': [earlyStopping]})
import numpy as np # SVG 표시에 필요한 라이브러리 임포트 from IPython.display import SVG from keras.utils.vis_utils import model_to_dot # 윈도우의 경우 다음을 추가 import os os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/' # 임의의 숫자로 더미 데이터를 준비 x_train = np.random.random((100, 6, 6, 1)) y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) x_test = np.random.random((20, 6, 6, 1)) y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10) # 합성곱 신경망 모델 작성 model = Sequential() model.add(Conv2D(filters=3, kernel_size=(3, 3), input_shape=(6, 6, 1), kernel_initializer='lecun_uniform', name='Conv2D_1')) model.add(Flatten(name='Flatten_1')) model.add(Dense(units=10, activation='softmax', name='Dense_1')) # 시퀀스 출력 SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg')) # 조기 종료 설정 earlyStopping = callbacks.EarlyStopping(monitor='val_loss', patience=5) # 모델 컴파일 model.compile(loss='mean_squared_error', optimizer='sgd') model.fit(x_train, y_train, batch_size=32, epochs=10, callbacks=[earlyStopping], validation_split=0.2)
def trainDAE(target, dataPath, refSampleInd, trainIndex, relevantMarkers, mode,
keepProb, denoise, loadModel, path):
sourceX = []
for i in np.arange(trainIndex.size - 1):
sourceIndex = np.delete(trainIndex, refSampleInd)[i]
source = dh.loadDeepCyTOFData(dataPath, sourceIndex, relevantMarkers,
mode)
numZerosOK = 1
toKeepS = np.sum((source.X == 0), axis=1) <= numZerosOK
if i == 0:
sourceX = source.X[toKeepS]
else:
sourceX = np.concatenate([sourceX, source.X[toKeepS]], axis=0)
# preProcess source
sourceX = np.log(1 + np.abs(sourceX))
numZerosOK = 1
toKeepT = np.sum((target.X == 0), axis=1) <= numZerosOK
inputDim = target.X.shape[1]
ae_encodingDim = 25
l2_penalty_ae = 1e-2
if denoise:
if loadModel:
from keras.models import load_model
autoencoder = load_model(
os.path.join(io.DeepLearningRoot(),
'savemodels/' + path + '/denoisedAE.h5'))
else:
# train de-noising auto encoder and save it.
trainTarget_ae = np.concatenate([sourceX, target.X[toKeepT]],
axis=0)
trainData_ae = trainTarget_ae * np.random.binomial(
n=1, p=keepProb, size=trainTarget_ae.shape)
input_cell = Input(shape=(inputDim, ))
encoded = Dense(ae_encodingDim,
activation='relu',
W_regularizer=l2(l2_penalty_ae))(input_cell)
encoded1 = Dense(ae_encodingDim,
activation='relu',
W_regularizer=l2(l2_penalty_ae))(encoded)
decoded = Dense(inputDim,
activation='linear',
W_regularizer=l2(l2_penalty_ae))(encoded1)
autoencoder = Model(input=input_cell, output=decoded)
autoencoder.compile(optimizer='rmsprop', loss='mse')
autoencoder.fit(trainData_ae,
trainTarget_ae,
nb_epoch=80,
batch_size=128,
shuffle=True,
validation_split=0.1,
verbose=0,
callbacks=[
mn.monitor(),
cb.EarlyStopping(monitor='val_loss',
patience=25,
mode='auto')
])
autoencoder.save(
os.path.join(io.DeepLearningRoot(),
'savemodels/' + path + '/denoisedAE.h5'))
del sourceX
plt.close('all')
return autoencoder
def trainModel(DS, x_train, y_train, x_valid, y_valid, x_test, y_test):
'''Build and Train (pretrained) ResNet50 Model'''
#Prepare Dataset for training
x_train, y_train, x_valid, y_valid, x_test, y_test = prepareDataset(
x_train, y_train, x_valid, y_valid, x_test, y_test)
# Define the parameters for ResNet50 model.
IMG_HEIGHT = 32
IMG_WIDTH = 32
IMG_DEPTH = 3
NB_EPOCHS = 20
no_of_class = noClass(DS)
print('Parameters are defined')
#Base Model
base_model = ResNet50(include_top=False,
pooling='avg',
weights='imagenet',
input_shape=(IMG_HEIGHT, IMG_WIDTH, IMG_DEPTH))
print('Base Model is created')
base_model.summary()
# Adding Dense Layers
x = base_model.output
# Adding a fully-connected layer
x = Dense(1024, activation='relu')(x)
# Addinf a softmax layer -- no of class
predictions = Dense(no_of_class, activation='softmax')(x)
# # Create new model
model = Model(inputs=base_model.input, outputs=predictions)
# Conv & pooling layers are not trainable
for layer in base_model.layers:
layer.trainable = False
print('Model is created')
# Compile the model.
model.compile(loss='categorical_crossentropy',
optimizer=optimizers.Adam(),
metrics=['acc'])
print('Model is compiled')
from keras import callbacks
#Define callbacks
reduce_learning = callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.2,
patience=2,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=2,
min_lr=0)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=7,
verbose=1,
mode='auto')
callbacks = [reduce_learning, early_stopping]
print('Call backs are defined')
print('\n\n>>>Model Trainig starts now')
# Train the the model
history = model.fit(x_train,
y_train,
epochs=NB_EPOCHS,
validation_data=(x_valid, y_valid),
callbacks=callbacks)
acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']
print('')
print(f'Model training accuracy: {acc[-1]}')
print(f'Model validation accuracy: {val_loss[-1]}')
print(f'Model training loss: {acc[-1]}')
print(f'Model validation loss: {val_loss[-1]}')
test_acc, test_precision, test_recall, test_fscore, hamming_loss = modelPerformance(
model, x_test, y_test)
results = {'train':[acc[-1], val_acc[-1], loss[-1], val_loss[-1]],\
'test':[test_acc, test_precision, test_recall, test_fscore, hamming_loss]}
return model, history, results
def train(self,
data=None,
trgt=None,
ifold=0,
hidden_neurons=None,
layer=1):
# Change elements equal to zero to one
if hidden_neurons is None:
hidden_neurons = [1]
for i in range(len(hidden_neurons)):
if hidden_neurons[i] == 0:
hidden_neurons[i] = 1
if (layer <= 0) or (layer > len(hidden_neurons)):
print("[-] Error: The parameter layer must be greater than zero and less " \
"or equal to the length of list hidden_neurons")
return -1
neurons_str = self.get_neurons_str(data, hidden_neurons[:layer])
model_str = os.path.join(
self.save_path,
self.prefix_str + "_{}_neurons".format(neurons_str))
trgt_sparse = np_utils.to_categorical(trgt.astype(int))
file_name = '%s_fold_%i_model.h5' % (model_str, ifold)
if os.path.exists(file_name):
if self.verbose:
print('File %s exists' % file_name)
# load model
file_name = '%s_fold_%i_model.h5' % (model_str, ifold)
classifier = load_model(
file_name,
custom_objects={
'%s' % self.parameters["HyperParameters"]["loss"]:
self.lossFunction
})
file_name = '%s_fold_%i_trn_desc.jbl' % (model_str, ifold)
trn_desc = joblib.load(file_name)
return ifold, classifier, trn_desc
train_id, test_id = self.CVO[ifold]
best_init = 0
best_loss = 999
classifier = []
trn_desc = {}
norm_data = self.normalize_data(data, ifold)
for i_init in range(self.parameters["HyperParameters"]["n_inits"]):
print(
'Neural Network - Layer: %i - Topology: %s - Fold %i of %i Folds - Init %i of %i Inits'
% (layer, neurons_str, ifold + 1,
self.parameters["HyperParameters"]["n_folds"], i_init + 1,
self.parameters["HyperParameters"]["n_inits"]))
model = Sequential()
for ilayer in range(layer):
if ilayer == 1:
#if bool(self.parameters["HyperParameters"]["dropout"]):
# model.add(Dropout(int(self.parameters["HyperParameters"]["dropout_parameter"])))
if self.parameters["HyperParameters"][
"regularization"] == "l1":
model.add(
Dense(units=hidden_neurons[ilayer],
input_dim=data.shape[1],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"],
kernel_regularizer=regularizers.l1(
self.parameters["HyperParameters"]
["regularization_parameter"])))
elif self.parameters["HyperParameters"][
"regularization"] == "l2":
model.add(
Dense(hidden_neurons[ilayer],
input_dim=data.shape[1],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"],
kernel_regularizer=regularizers.l2(
self.parameters["HyperParameters"]
["regularization_parameter"])))
else:
model.add(
Dense(
hidden_neurons[ilayer],
input_dim=data.shape[1],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"]))
else:
#if bool(self.parameters["HyperParameters"]["dropout"]):
# model.add(Dropout(int(self.parameters["HyperParameters"]["dropout_parameter"])))
if self.parameters["HyperParameters"][
"regularization"] == "l1":
model.add(
Dense(units=hidden_neurons[ilayer],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"],
kernel_regularizer=regularizers.l1(
self.parameters["HyperParameters"]
["regularization_parameter"])))
elif self.parameters["HyperParameters"][
"regularization"] == "l2":
model.add(
Dense(hidden_neurons[ilayer],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"],
kernel_regularizer=regularizers.l2(
self.parameters["HyperParameters"]
["regularization_parameter"])))
else:
model.add(
Dense(
hidden_neurons[ilayer],
kernel_initializer=self.parameters[
"HyperParameters"]["kernel_initializer"]))
model.add(
Activation(self.parameters["HyperParameters"]
["hidden_activation_function"]))
# Add Output Layer
model.add(
Dense(units=trgt_sparse.shape[1],
kernel_initializer=self.parameters["HyperParameters"]
["kernel_initializer"]))
model.add(
Activation(self.parameters["HyperParameters"]
["classifier_output_activation_function"]))
model.compile(
loss=self.lossFunction,
optimizer=self.optmizer,
metrics=self.parameters["HyperParameters"]["metrics"])
# Train model
earlyStopping = callbacks.EarlyStopping(
monitor=self.parameters["callbacks"]["EarlyStopping"]
["monitor"],
patience=self.parameters["callbacks"]["EarlyStopping"]
["patience"],
verbose=self.verbose,
mode='auto')
class_weights = getGradientWeights(trgt[train_id])
init_trn_desc = model.fit(
norm_data[train_id],
trgt_sparse[train_id],
epochs=self.parameters["HyperParameters"]["n_epochs"],
batch_size=self.parameters["HyperParameters"]["batch_size"],
callbacks=[earlyStopping],
verbose=self.verbose,
validation_data=(norm_data[test_id], trgt_sparse[test_id]),
shuffle=True,
class_weight=class_weights)
if np.min(init_trn_desc.history['val_loss']) < best_loss:
best_init = i_init
trn_desc['best_init'] = best_init
best_loss = np.min(init_trn_desc.history['val_loss'])
classifier = model
trn_desc['epochs'] = init_trn_desc.epoch
for imetric in range(
len(self.parameters["HyperParameters"]["metrics"])):
if self.parameters["HyperParameters"]["metrics"][
imetric] == 'accuracy':
metric = 'acc'
else:
metric = self.parameters["HyperParameters"]["metrics"][
imetric]
trn_desc[metric] = init_trn_desc.history[metric]
trn_desc['val_' + metric] = init_trn_desc.history['val_' +
metric]
trn_desc['loss'] = init_trn_desc.history['loss']
trn_desc['val_loss'] = init_trn_desc.history['val_loss']
# save model
file_name = '%s_fold_%i_model.h5' % (model_str, ifold)
classifier.save(file_name)
file_name = '%s_fold_%i_trn_desc.jbl' % (model_str, ifold)
joblib.dump([trn_desc], file_name, compress=9)
return ifold, classifier, trn_desc
def train_Inception(train_df, test_df, label_col, batch, weights, mode):
#Top 3 categorical accuracy method
def top_3_categorical_accuracy(y_true, y_pred):
return metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
#Top 2 categorical accuracy method
def top_2_categorical_accuracy(y_true, y_pred):
return metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)
#Adam optimizer
#Default: lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False
adam = optimizers.Adam(lr=0.00001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
#Check if binary or multiclass classification and sets parameters
if mode == 0:
output = 1
final_activation = 'sigmoid'
optimizer_mode = adam
loss_mode = 'binary_crossentropy'
metrics_mode = ['binary_accuracy']
flow_mode = "binary"
classes = [-1, 1]
else:
output = 7
final_activation = 'softmax'
optimizer_mode = adam
loss_mode = 'categorical_crossentropy'
metrics_mode = [
'categorical_accuracy', top_2_categorical_accuracy,
top_3_categorical_accuracy
]
flow_mode = "categorical"
classes = [-1, 0, 1, 2, 3, 4, 5]
#Create a base model using the built-in Inceptionv3 model in Keras
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3),
pooling='avg')
#Classifier for the model
x = base_model.output
res_predictions = Dense(units=output, activation=final_activation)(x)
#Create a functional API model using Inception v3 and a new classifier
transfer = Model(inputs=base_model.input, outputs=res_predictions)
#Optional freezing of layers if not enough GPU memory
#for layer in transfer.layers[:52]: #Freeze block 1
#layer.trainable = False
# Compile model
# Optimiser parameter: adam, Loss function: Binary Crossentropy, Performance metric: Binary Accuracy
transfer.compile(optimizer=optimizer_mode,
loss=loss_mode,
metrics=metrics_mode)
#Realtime image augmentation
#Rescale: rescales RGB values to 0-1
#, rotation_range = 40, width_shift_range=0.2, height_shift_range=0.2, fill_mode='nearest'
train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input,
validation_split=0.25)
test_datagen = ImageDataGenerator(
preprocessing_function=preprocess_input
) #We don't modify the testing images as we are only doing predictions
#Takes dataframe input and images directory and generate batches of augmented data
train_generator = train_datagen.flow_from_dataframe(dataframe=train_df,
directory=train_dest,
x_col="file_name",
y_col=label_col,
has_ext=False,
batch_size=batch,
seed=42,
shuffle=True,
class_mode=flow_mode,
target_size=(299, 299),
subset="training",
classes=classes)
valid_generator = train_datagen.flow_from_dataframe(dataframe=train_df,
directory=train_dest,
x_col="file_name",
y_col=label_col,
has_ext=False,
batch_size=batch,
seed=42,
shuffle=True,
class_mode=flow_mode,
target_size=(299, 299),
subset="validation",
classes=classes)
test_generator = test_datagen.flow_from_dataframe(
dataframe=test_df,
directory=test_dest,
x_col="file_name",
y_col=None,
has_ext=False,
batch_size=1,
seed=42,
shuffle=False,
class_mode=None,
target_size=(299, 299)) #Returns images
test_index = test_datagen.flow_from_dataframe(
dataframe=test_df,
directory=test_dest,
x_col="file_name",
y_col=label_col,
has_ext=False,
batch_size=1,
seed=42,
shuffle=False,
class_mode=flow_mode,
target_size=(299, 299)) #Returns images
#Obtain steps required for training, evaluating and testing
step_size_train = train_generator.n // train_generator.batch_size
step_size_valid = valid_generator.n // valid_generator.batch_size
step_size_test = test_generator.n // test_generator.batch_size
#Callbacks
csv_logger = callbacks.CSVLogger('./logs/' + label_col +
'_training_log.csv',
separator=',',
append=False)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=3,
verbose=0,
mode='auto')
# Checks how to set class_weight parameter in fit_generator based on user input
if weights == 'auto':
index = train_generator.class_indices
calc_class = class_weight.compute_class_weight(
'balanced', np.unique(train_generator.classes),
train_generator.classes)
class_weights = dict(zip(index.values(), calc_class))
elif weights == 'nan_suppress':
index = train_generator.class_indices
calc_class = class_weight.compute_class_weight(
'balanced', np.unique(train_generator.classes),
train_generator.classes)
calc_class[0] = 0.
class_weights = dict(zip(index.values(), calc_class))
elif weights == 'equal':
if mode == 1:
class_weights = {0: 1., 1: 1., 2: 1., 3: 1., 4: 1., 5: 1., 6: 1.}
else:
class_weights = {0: 1., 1: 1.}
else:
class_weights = weights
#Train model on generated data via batches
train_history = transfer.fit_generator(
train_generator,
epochs=50,
class_weight=class_weights,
steps_per_epoch=step_size_train,
validation_data=valid_generator,
validation_steps=step_size_valid,
verbose=1,
callbacks=[csv_logger, early_stopping],
workers=10)
# Evaluate model and assign loss and accuracy scores
score = transfer.evaluate_generator(generator=valid_generator,
steps=step_size_valid,
verbose=1)
# Predict output
test_generator.reset()
prediction = transfer.predict_generator(test_generator,
steps=step_size_test,
verbose=1) #Returns numpy array
#Obtain predicted labels in an array through binary threshold or argmax
if mode == 0:
predicted_class_indices = [1 if x >= 0.5 else 0 for x in prediction]
else:
predicted_class_indices = np.argmax(prediction,
axis=1) #argmax across rows
#Map predicted labels to their unique filenames
labels = (
train_generator.class_indices
) #Get the class label indices (so we know what 0 and 1 refers to)
labels = dict((v, k) for k, v in labels.items()) #Reverse indices
predictions = [labels[k] for k in predicted_class_indices
] #Get all predictions (class) from dict
filenames = test_generator.filenames #Get all filenames of predictions
results = pd.DataFrame({
"Filename": filenames,
"Predictions": predictions
}) #Save filename and predictions to a dataframe
return results, train_history, score
def createEarlyStopping():
return callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, mode='auto')
def train_CNN(train_df, test_df, label_col, batch, weights, mode):
#Top 3 categorical accuracy method
def top_3_categorical_accuracy(y_true, y_pred):
return metrics.top_k_categorical_accuracy(y_true, y_pred, k=3)
#Top 2 categorical accuracy method
def top_2_categorical_accuracy(y_true, y_pred):
return metrics.top_k_categorical_accuracy(y_true, y_pred, k=2)
#Adam optimizer
#Default: lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False
adam = optimizers.Adam(lr=0.00001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.0,
amsgrad=False)
#Check if binary or multiclass classification and sets parameters
if mode == 0:
output = 1
#sigmoid, softmax, elu, softplus, relu, tanh
final_activation = 'sigmoid'
optimizer_mode = adam
#binary_crossentropy, mean_squared_error, hinge, kullback_leibler_divergence, poisson
loss_mode = 'binary_crossentropy'
metrics_mode = ['binary_accuracy']
flow_mode = "binary"
classes = [-1, 1]
else:
output = 7
final_activation = 'softmax'
optimizer_mode = adam
loss_mode = 'categorical_crossentropy'
metrics_mode = [
'categorical_accuracy', top_2_categorical_accuracy,
top_3_categorical_accuracy
]
flow_mode = "categorical"
classes = [-1, 0, 1, 2, 3, 4, 5]
classifier = Sequential(
) #Create Sequential object which allows layers to be used
# Convolution layer with Non Linearity
# To extract features from input
# Filters: 32, Filter shape: 3X3, Input shape: 256X256, RGB(3), Activation function: Rectifier Linear Unit
classifier.add(
Conv2D(32, (3, 3), input_shape=(256, 256, 3), activation='relu'))
# Pooling layer using Max Pooling
# To reduce the size of images and total number of nodes
# Pool size: 2X2
classifier.add(MaxPooling2D(pool_size=(2, 2)))
# Dropout for regularisation
#classifier.add(Dropout(0.2))
#More convolution and pooling layers
classifier.add(Conv2D(32, (3, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))
#classifier.add(Dropout(0.2))
classifier.add(Conv2D(64, (3, 3), activation='relu'))
classifier.add(MaxPooling2D(pool_size=(2, 2)))
#classifier.add(Dropout(0.2))
# Flattening
# Convert 3D feature map to 1D feature vector
classifier.add(Flatten())
# Fully Connected/ Hidden layer
# A fully connected layer
# Nodes: 128, Activation function: Rectifier Linear Unit
classifier.add(Dense(units=128, activation='relu'))
#classifier.add(Dropout(0.2))
# Output layer
# One node for binary output
# Activation function: Sigmoid
classifier.add(Dense(units=output, activation=final_activation))
# Compile CNN
# Optimiser parameter: adam, Loss function: Binary Crossentropy, Performance metric: Binary Accuracy
classifier.compile(optimizer=optimizer_mode,
loss=loss_mode,
metrics=metrics_mode)
#Realtime image augmentation
#Rescale: rescales RGB values to 0-1
#, rotation_range = 40, width_shift_range=0.2, height_shift_range=0.2, fill_mode='nearest'
train_datagen = ImageDataGenerator(rescale=1. / 255,
validation_split=0.25,
shear_range=0.2,
zoom_range=0.2,
horizontal_flip=True)
test_datagen = ImageDataGenerator(
rescale=1. / 255
) #We don't modify the testing images as we are only doing predictions
#Takes dataframe input and images directory and generate batches of augmented data
train_generator = train_datagen.flow_from_dataframe(dataframe=train_df,
directory=train_dest,
x_col="file_name",
y_col=label_col,
has_ext=False,
batch_size=batch,
seed=42,
shuffle=True,
class_mode=flow_mode,
target_size=(256, 256),
subset="training",
classes=classes)
valid_generator = train_datagen.flow_from_dataframe(dataframe=train_df,
directory=train_dest,
x_col="file_name",
y_col=label_col,
has_ext=False,
batch_size=batch,
seed=42,
shuffle=True,
class_mode=flow_mode,
target_size=(256, 256),
subset="validation",
classes=classes)
test_generator = test_datagen.flow_from_dataframe(
dataframe=test_df,
directory=test_dest,
x_col="file_name",
y_col=None,
has_ext=False,
batch_size=1,
seed=42,
shuffle=False,
class_mode=None,
target_size=(256, 256)) #Returns images
#test_generator = test_datagen.flow_from_directory(directory = test_dest, batch_size = 1, seed = 42, shuffle = False, class_mode = None, target_size = (256, 256)) #Returns images
#Obtain steps required for training, evaluating and testing
step_size_train = train_generator.n // train_generator.batch_size
step_size_valid = valid_generator.n // valid_generator.batch_size
step_size_test = test_generator.n // test_generator.batch_size
#Callbacks
tensorboard = callbacks.TensorBoard(log_dir='./logs',
histogram_freq=0,
write_graph=True,
write_images=True)
csv_logger = callbacks.CSVLogger('./logs/' + label_col +
'_training_log.csv',
separator=',',
append=False)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=3,
verbose=0,
mode='auto')
# Checks how to set class_weight parameter in fit_generator based on user input
if weights == 'auto':
index = train_generator.class_indices
calc_class = class_weight.compute_class_weight(
'balanced', np.unique(train_generator.classes),
train_generator.classes)
class_weights = dict(zip(index.values(), calc_class))
elif weights == 'nan_suppress':
index = train_generator.class_indices
calc_class = class_weight.compute_class_weight(
'balanced', np.unique(train_generator.classes),
train_generator.classes)
calc_class[0] = 0.
class_weights = dict(zip(index.values(), calc_class))
elif weights == 'equal':
if mode == 1:
class_weights = {0: 1., 1: 1., 2: 1., 3: 1., 4: 1., 5: 1., 6: 1.}
else:
class_weights = {0: 1., 1: 1.}
else:
class_weights = weights
#Train model on generated data via batches
train_history = classifier.fit_generator(
train_generator,
epochs=100,
class_weight=class_weights,
steps_per_epoch=step_size_train,
validation_data=valid_generator,
validation_steps=step_size_valid,
verbose=1,
callbacks=[csv_logger, early_stopping],
workers=10)
# Save weights
classifier.save_weights('CNN.h5')
# Evaluate model and assign loss and accuracy scores
score = classifier.evaluate_generator(generator=valid_generator,
steps=step_size_valid,
verbose=1)
# Predict output
test_generator.reset()
prediction = classifier.predict_generator(test_generator,
steps=step_size_test,
verbose=1) #Returns numpy array
#Obtain predicted labels in an array through binary threshold or argmax
if mode == 0:
predicted_class_indices = [1 if x >= 0.5 else 0 for x in prediction]
else:
predicted_class_indices = np.argmax(prediction,
axis=1) #argmax across rows
#Map predicted labels to their unique filenames
labels = (
train_generator.class_indices
) #Get the class label indices (so we know what 0 and 1 refers to)
labels = dict((v, k) for k, v in labels.items()) #Reverse indices
predictions = [labels[k] for k in predicted_class_indices
] #Get all predictions (class) from dict
filenames = test_generator.filenames #Get all filenames of predictions
results = pd.DataFrame({
"Filename": filenames,
"Predictions": predictions
}) #Save filename and predictions to a dataframe
return results, train_history, score
def build_LSTM_model(trainData, trainBatches, testData, testBatches,
windowSize, class_count, numCalls, batch_size):
# Specify number of units
# https://stackoverflow.com/questions/37901047/what-is-num-units-in-tensorflow-basiclstmcell#39440218
num_units = 128
embedding_size = 256
# https://keras.io/callbacks/#earlystopping
early_stop = cb.EarlyStopping(monitor='sparse_categorical_accuracy',
min_delta=0.0001,
patience=3)
# We need to add an embedding layer because LSTM (at this moment) that the API call indices (numbers)
# are of some mathematical significance. E.g., system call 2 is "closer" to system calls 3 and 4.
# But system call numbers have nothing to do with their semantic meaning and relation to other
# system calls. So we transform it using an embedding layer so the LSTM can figure these relationships
# out for itself.
# https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html
# https://stackoverflow.com/questions/40695452/stateful-lstm-with-embedding-layer-shapes-dont-match
api_count = numCalls + 1 # +1 because 0 is our padding number
inp = Input(shape=(windowSize, ))
emb = Embedding(input_dim=api_count,
output_dim=256,
input_length=windowSize)(inp)
# https://keras.io/layers/recurrent/#lstm
# model.add(LSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False))
#TODO - GPU stuffs
# model.add(CuDNNLSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False))
# From malconv paper
filt = Conv1D(filters=64,
kernel_size=3,
strides=1,
use_bias=True,
activation='relu',
padding='valid')(emb)
attn = Conv1D(filters=64,
kernel_size=3,
strides=1,
use_bias=True,
activation='sigmoid',
padding='valid')(emb)
gated = Multiply()([filt, attn])
drop = Dropout(0.5)(gated)
feat = GlobalMaxPooling1D()(drop)
dense = Dense(128, activation='relu')(feat)
outp = Dense(class_count, activation='sigmoid')(dense)
model = Model(inp, outp)
# Which optimizer to use
# https://keras.io/optimizers/
opt = optimizers.RMSprop(lr=0.01, decay=0.001)
# https://keras.io/models/model/#compile
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=opt,
# Metrics to print
# We use sparse_categorical_accuracy as opposed to categorical_accuracy
# because: https://stackoverflow.com/questions/44477489/keras-difference-between-categorical-accuracy-and-sparse-categorical-accuracy
# I.e., since we don't use hot-encoding, we use sparse_categorical_accuracy
metrics=['sparse_categorical_accuracy'])
# https://keras.io/models/model/#fit_generator
hist = model.fit_generator(
# Data to train
trainData,
# Use multiprocessing because python Threading isn't really
# threading: https://docs.python.org/3/glossary.html#term-global-interpreter-lock
use_multiprocessing=True,
# Number of steps per epoch (this is how we train our large
# number of samples dataset without running out of memory)
steps_per_epoch=trainBatches,
#TODO
# Number of epochs
epochs=100,
# Validation data (will not be trained on)
validation_data=testData,
validation_steps=testBatches,
# Do not shuffle batches.
shuffle=False,
# List of callbacks to be called while training.
callbacks=[early_stop])
return model, hist
authors_max_length=authors_max_length,
authors_input_size=authors_input_size,
authors_embedding_matrix=authors_embedding_matrix,
node_2_vec_emb_size=NODE_2_VEC_EMB_SIZE,
regularization=regularization,
static_input_size=STATIC_INPUT_SIZE,
dropout=DROPOUT,
opt=opt)
callbacks_list = [
callbacks.TensorBoard(log_dir=TENSORBOARD_LOGS_DIR,
histogram_freq=0,
embeddings_freq=1,
write_graph=True,
write_images=False),
callbacks.EarlyStopping(monitor='acc', patience=10),
callbacks.ModelCheckpoint(filepath=model_outfile,
monitor='val_loss',
save_best_only=True),
callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2)
]
x_train_input = [
x_train_abstracts_padded, x_train_titles_padded, x_train_authors_padded,
x_train_static
]
x_val_input = [
x_val_abstracts_padded, x_val_titles_padded, x_val_authors_padded,
x_val_static
]
# 使得validation数据量大小为batch_size的整数陪
nb_val_samples = len(y_val) - len(y_val) % batch_size
model = get_model(shape)
print(model.summary())
# 根据validation loss保存最优模型
save_best = callbacks.ModelCheckpoint('best_model.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
# 如果训练持续没有validation loss的提升, 提前结束训练
early_stop = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=15,
verbose=0,
mode='auto')
tbCallback = callbacks.TensorBoard(log_dir='./Graph', write_graph=True)
callbacks_list = [early_stop, save_best, tbCallback]
history = model.fit_generator(
batch_generator(X_train,
y_train,
batch_size,
shape,
training=True,
data_dir=data_path),
steps_per_epoch=samples_per_epoch,
validation_steps=nb_val_samples // batch_size,
validation_data=batch_generator(X_val,
#n_dim = combined_feat.dimension()
#scale = pre.MinMaxScaler(feature_range=(-1, 1))
#scale.fit(np.concatenate(combined_data))
# Create chunked data.
#all_data = scale.transform(np.concatenate(combined_data))
all_data = np.load('all_data.npy')
n_dim = all_data.shape[1]
n_chunks = args.n_chunks
n_data = all_data.shape[0]
n_batch = n_data // n_chunks
chunked_data = [all_data[i:i + n_batch] for i in range(0, n_data, n_batch)]
k_folds = KFold(n_splits=n_folds, shuffle=True)
calls = [callbacks.EarlyStopping(patience=30, restore_best_weights=True)]
train_scores = np.zeros(n_folds)
test_scores = np.zeros(n_folds)
hde_timescales = []
msm_timescales = []
for i in range(n_folds):
#train_val_data = [chunked_data[j] for j in train_idx]
#test_data = [chunked_data[j] for j in test_idx]
print('Processing fold {}'.format(i))
train_data, test_data = train_test_split(chunked_data, test_size=0.5)
hde = HDE(
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(200,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(10,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',
patience=10,
restore_best_weights=True)
model.compile(optimizer=opt.Adam(),
loss="categorical_crossentropy",
metrics=['acc'])
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
validation_split=(1 / 6),
callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
# print("y_train shape :", y_train.shape)
# print('x_val shape :', x_val.shape)
# print('y_val shape :', y_val.shape)
# print("x_test shape :", x_test.shape)
# print("y_test shape :", y_test.shape)
print("x_train shape :", x_train.shape, x_train1.shape)
print("y_train shape :", y_train.shape)
print('x_val shape :', x_val.shape, x_val1.shape)
print('y_val shape :', y_val.shape)
print("x_test shape :", x_test.shape, x_test1.shape)
print("y_test shape :", y_test.shape)
model = IN_net.model()
# kcallback
earlyStopping6 = kcallbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=1, mode='auto')
saveBestModel6 = kcallbacks.ModelCheckpoint(filepath='ckpt/best.h5', monitor='val_loss', verbose=1,save_best_only=True, mode='auto')
# reduce_lr = ReduceLROnPlateau(monitor='val_loss',factor=0.5, patience=20, mode='auto')
print("----------------------------training-begin------------------------------")
tic6 = time.clock()
History = model.fit([x_train, x_train1], y_train, nb_epoch =nb_epoch, batch_size=16,
callbacks=[earlyStopping6, saveBestModel6], validation_data=([x_val, x_val1], y_val))
toc6 = time.clock()
# model.save("models/two_channel_IN.h5")
# model.save("models/spa_network.h5")
model.save("models/test3_best.h5")
scores = model.evaluate([x_val, x_val1], y_val, batch_size=25)
print('Test score:', scores[0])
def run(_run, image_shape, data_dir, train_shuffle, dataset_train_seed,
valid_shuffle, dataset_valid_seed, classes, class_mode, class_weight,
architecture, weights, batch_size, base_layers, pooling, dense_layers,
predictions_activation, metrics, loss, device, opt_params, dropout_p,
resuming_from_ckpt_file, steps_per_epoch, epochs, validation_steps,
workers, use_multiprocessing, initial_epoch, early_stop_patience,
tensorboard_tag, first_trainable_layer):
report_dir = _run.observers[0].dir
g = ImageDataGenerator(
horizontal_flip=True,
vertical_flip=True,
samplewise_center=True,
samplewise_std_normalization=True,
zoom_range=45,
rotation_range=.2,
height_shift_range=.2,
width_shift_range=.2,
fill_mode='reflect',
preprocessing_function=get_preprocess_fn(architecture))
if isinstance(classes, int):
classes = sorted(os.listdir(os.path.join(data_dir, 'train')))[:classes]
train_data = g.flow_from_directory(os.path.join(data_dir, 'train'),
target_size=image_shape[:2],
classes=classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=train_shuffle,
seed=dataset_train_seed)
valid_data = g.flow_from_directory(os.path.join(data_dir, 'valid'),
target_size=image_shape[:2],
classes=classes,
class_mode=class_mode,
batch_size=batch_size,
shuffle=valid_shuffle,
seed=dataset_valid_seed)
if class_weight == 'balanced':
class_weight = get_class_weights(train_data.classes)
if steps_per_epoch is None:
steps_per_epoch = ceil(train_data.n / batch_size)
if validation_steps is None:
validation_steps = ceil(valid_data.n / batch_size)
with tf.device(device):
print('building...')
model = build_gram_model(image_shape,
architecture=architecture,
weights=weights,
dropout_p=dropout_p,
classes=train_data.num_classes,
base_layers=base_layers,
pooling=pooling,
dense_layers=dense_layers,
predictions_activation=predictions_activation)
model.summary()
layer_names = [l.name for l in model.layers]
if first_trainable_layer:
if first_trainable_layer not in layer_names:
raise ValueError('%s is not a layer in the model: %s' %
(first_trainable_layer, layer_names))
for layer in model.layers:
if layer.name == first_trainable_layer:
break
layer.trainable = False
model.compile(optimizer=optimizers.Adam(**opt_params),
metrics=metrics,
loss=loss)
if resuming_from_ckpt_file:
print('re-loading weights...')
model.load_weights(resuming_from_ckpt_file)
print('training from epoch %i...' % initial_epoch)
try:
model.fit_generator(
train_data,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=2,
validation_data=valid_data,
validation_steps=validation_steps,
initial_epoch=initial_epoch,
class_weight=class_weight,
workers=workers,
use_multiprocessing=use_multiprocessing,
callbacks=[
# callbacks.LearningRateScheduler(lambda epoch: .5 ** (epoch // 10) * opt_params['lr']),
callbacks.TerminateOnNaN(),
callbacks.ReduceLROnPlateau(min_lr=1e-10,
patience=int(
early_stop_patience // 3)),
callbacks.EarlyStopping(patience=early_stop_patience),
callbacks.TensorBoard(os.path.join(report_dir,
tensorboard_tag),
batch_size=batch_size),
callbacks.ModelCheckpoint(os.path.join(
report_dir, 'weights.h5'),
save_best_only=True,
verbose=1),
])
except KeyboardInterrupt:
print('interrupted by user')
else:
print('done')
else:
decoder_final=Conv2D(1,(3,3),activation='sigmoid',data_format='channels_first',padding='same')(decoder_layer_2)
logging.debug(str(decoder_final.shape))
autoencoder=Model(input_img, decoder_final)
autoencoder.compile(optimizer=args.opt_name, loss=args.loss_name)
#start training
logging.info("Training with "+image_type+" images")
logging.info("Current training set is made from "+str(len(train_data_list))+" files and has "+str(get_num_samples(train_data_list,image_type))+" examples")
early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=3) # Well this is sorta useless here, since nb epochs is actually set to 1 in model.fit and the number of epochs here is the number of time the outer loop runs
tensorboard=callbacks.TensorBoard(log_dir='../logs', histogram_freq=1, batch_size=32, write_graph=True, write_grads=True, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)
training_history=train_histories()
num_epochs=10
patience=3 #number of epochs where we see no improvement after which we stop training
last_epoch_val_loss=1000 #arbitrarily large number
perc_decrease_per_epoch=10
epochwise_loss_history=[]
batchwise_loss_history=[]
epochwise_val_loss_history=[]
batchwise_val_loss_history=[]
epoch_count=0
width_shift_range=0.05,
height_shift_range=0.05,
shear_range=0.,
zoom_range=0.1,
horizontal_flip=True,
vertical_flip=False,
dim_ordering='th')
datagen.fit(X_train)
if zca_flag:
print 'zca whitening enabled, saving zca matrix...'
datagen.principal_components.dump('zca_matrix')
print 'training model...'
model.fit_generator(datagen.flow(X_train, y_train, batch_size=128, shuffle=True),
samples_per_epoch=len(X_train),
nb_val_samples=len(X_test),
class_weight={0: 1, 1: 1},
nb_epoch=200,
verbose=1,
validation_data=datagen.flow(X_val, y_val, shuffle=True),
callbacks=[callbacks.EarlyStopping(monitor='val_acc', patience=20, verbose=1, mode='auto')])
print 'training complete'
print 'evaluating model...'
score = model.evaluate(X_test, y_test, batch_size=32, verbose=1)
print 'test accuracy : ' + str(score[1])
print 'saving architecture and weights...'
json_string = model.to_json()
open('model.json', 'w').write(json_string)
model.save_weights('weights.h5')
def train(data_folder, fn_sequences, batch_size, n_epochs, output_folder):
with open(os.path.join(data_folder, fn_sequences), 'r') as f:
seqs = f.read().split('\n')
tokenizer = Tokenizer()
tokenizer.fit_on_texts(seqs)
sequences = tokenizer.texts_to_sequences(seqs)
vocab_size = len(tokenizer.word_index) + 1
print(vocab_size)
from keras.utils import to_categorical
sequences = np.array(sequences)
X, y = sequences[:, :-1], sequences[:, -1]
y = keras.utils.to_categorical(y, num_classes=vocab_size)
seq_length = X.shape[1]
# define model
model = Sequential()
model.add(Embedding(vocab_size, 50, input_length=seq_length))
model.add(LSTM(100, return_sequences=True))
model.add(LSTM(100))
model.add(Dense(100, activation='relu'))
model.add(Dense(vocab_size, activation='softmax'))
print(model.summary())
model_checkpoint = callbacks.ModelCheckpoint("my_checkpoint.h5",
save_best_only=True)
early_stopping = callbacks.EarlyStopping(patience=50)
# compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# fit model
# start an Azure ML run
run = Run.get_context()
class LogRunMetrics(callbacks.Callback):
# callback at the end of every epoch
def on_epoch_end(self, epoch, log):
# log a value repeated which creates a list
run.log('Loss', log['loss'])
# model.fit(X, y, batch_size=256, epochs=200, callbacks=[early_stopping, model_checkpoint])
history = model.fit(
X,
y,
batch_size=batch_size,
epochs=n_epochs,
verbose=2,
callbacks=[LogRunMetrics(), early_stopping, model_checkpoint])
# log a single value
plt.figure(figsize=(6, 3))
plt.title('Goethe LM with Keras MLP ({} epochs)'.format(n_epochs),
fontsize=14)
plt.plot(history.history['loss'], 'r--', label='Loss', lw=4, alpha=0.5)
plt.legend(fontsize=12)
plt.grid(True)
# log an image
run.log_image('Loss', plot=plt)
# create a ./outputs/model folder in the compute target
# files saved in the "./outputs" folder are automatically uploaded into run history
folder_model = os.path.join(output_folder, 'model_goethe')
os.makedirs(folder_model, exist_ok=True)
# save the tokenizer
pickle.dump(tokenizer,
open(os.path.join(folder_model, 'tokenizer.pkl', 'wb')))
# save the model to file
# model.save('model_goethe_generator.h5')
# serialize NN architecture to JSON
model_json = model.to_json()
# save model JSON
with open(os.path.join(folder_model, 'model.json'), 'w') as f:
f.write(model_json)
# save model weights
model.save_weights(os.path.join(folder_model, 'model_weights.h5'))
print("model saved in folder " + folder_model)
return model
es = keras.callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=5,
verbose=0, mode='auto')
'''
folds = 1
rmse_avg = []
r2_avg = []
nrmse_avg = []
loss_avg = []
count = 0
for i in tqdm(range(folds)):
es = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=7,
verbose=0,
mode='auto',
restore_best_weights=True)
X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.2)
print(X_train.shape, X_test.shape, Y_train.shape, Y_test.shape)
model = Sequential()
model.add(
layers.Dense(256, activation='relu', input_shape=(X_train.shape[1], )))
#model.add(layers.Dropout(0.3))
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
#model.add(layers.Dropout(0.3))
model.add(layers.Dense(32, activation='relu'))
#model.add(layers.Dropout(0.3))
model.add(layers.Dense(16, activation='relu'))
model.add(layers.Dense(8, activation='relu'))
pred = self.model.predict(self.validation_data[0])
prediction = np.round(np.asarray(pred))
target = self.validation_data[1]
precision = sklm.precision_score(target, prediction)
recall = sklm.recall_score(target, prediction)
f1_score = sklm.f1_score(target, prediction)
avg_precision = sklm.average_precision_score(target,prediction,average='weighted')
print("Metrics-val:",
"precision=", precision, ",recall=", recall,
",f1=", f1_score, ",avg_prec=", avg_precision)
metrics = Metrics(x_tr,y_tr)
#-------------------------------------------------------------------
# early stopping
earlyStopping=CB.EarlyStopping(monitor='val_loss', patience=10, mode='auto')
# start learning
model.fit(x_tr, y_tr, validation_data=(x_va, y_va),
epochs=epoch_count,
batch_size=batch_size,
#callbacks=[metrics, earlyStopping],
callbacks=[earlyStopping],
verbose=2)
np.set_printoptions(threshold=np.inf)
print("\nPrediction on validation set:")
va_predictions = model.predict(x_va, batch_size, 1)
#print("VA_PREDICTIONS:",va_predictions)
print("\nPrediction on training set:")
tr_predictions = model.predict(x_tr, batch_size,1)
# print the label against the rounded prediction for easy comparison
mode='easgd', sync_every=args.sync_every,
worker_optimizer=args.worker_optimizer,
elastic_force=args.elastic_force/(comm.Get_size()-1),
elastic_lr=args.elastic_lr,
elastic_momentum=args.elastic_momentum)
else:
algo = Algo(args.optimizer, loss=args.loss, validate_every=validate_every,
sync_every=args.sync_every, worker_optimizer=args.worker_optimizer)
# Most Keras callbacks are supported
callbacks = []
callbacks.append( cbks.ModelCheckpoint( '_'.join([
model_name,args.trial_name,"mpi_learn_result.h5"]),
monitor='val_loss', verbose=1 ) )
if args.early_stopping is not None:
callbacks.append( cbks.EarlyStopping( patience=args.early_stopping,
verbose=1 ) )
# Creating the MPIManager object causes all needed worker and master nodes to be created
manager = MPIManager( comm=comm, data=data, algo=algo, model_builder=model_builder,
num_epochs=args.epochs, train_list=train_list, val_list=val_list,
num_masters=args.masters, synchronous=args.synchronous,
callbacks=callbacks, verbose=args.verbose )
# Process 0 launches the training procedure
if comm.Get_rank() == 0:
print (algo)
t_0 = time()
histories = manager.process.train()
delta_t = time() - t_0
manager.free_comms()
def run_model(
train_generator,
validation_generator,
dl_model,
output_folder,
instance_name,
image_size,
aggregate_value,
nb_labels,
nb_epochs,
nb_training_image,
nb_validation_image,
batch_size,
dropout,
network,
learning_rate,
learning_rate_decay,
):
"""Run deep learning `dl_model` starting from training and validation data
generators, depending on a range of hyperparameters
Parameters
----------
train_generator : generator
Training data generator
validation_generator : generator
Validation data generator
dl_model : str
Name of the addressed research problem (*e.g.* `feature_detection` or
`semantic_segmentation`)
output_folder : str
Name of the folder where the trained model will be stored on the file
system
instance_name : str
Name of the instance
image_size : int
Size of images, in pixel (height=width)
aggregate_value : str
Label aggregation policy (either `full` or `aggregated`)
nb_labels : int
Number of labels into the dataset
nb_epochs : int
Number of epochs during which models will be trained
nb_training_image : int
Number of images into the training dataset
nb_validation_image : int
Number of images into the validation dataset
batch_size : int
Number of images into each batch
dropout : float
Probability of keeping a neuron during dropout phase
network : str
Neural network architecture (*e.g.* `simple`, `vgg`, `inception`)
learning_rate : float
Starting learning rate
learning_rate_decay : float
Learning rate decay
Returns
-------
dict
Dictionary that summarizes the instance and the corresponding model
performance (measured by validation accuracy)
"""
if dl_model == "featdet":
net = FeatureDetectionNetwork(
network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network,
)
loss_function = "binary_crossentropy"
elif dl_model == "semseg":
net = SemanticSegmentationNetwork(
network_name=instance_name,
image_size=image_size,
nb_channels=3,
nb_labels=nb_labels,
architecture=network,
)
loss_function = "categorical_crossentropy"
else:
logger.error(("Unrecognized model: %s. Please choose amongst %s",
dl_model, AVAILABLE_MODELS))
sys.exit(1)
model = Model(net.X, net.Y)
opt = Adam(lr=learning_rate, decay=learning_rate_decay)
metrics = ["acc", iou, dice_coef]
model.compile(loss=loss_function, optimizer=opt, metrics=metrics)
# Model training
steps = max(nb_training_image // batch_size, 1)
val_steps = max(nb_validation_image // batch_size, 1)
checkpoint_files = [
item for item in os.listdir(output_folder)
if "checkpoint-epoch" in item
]
if len(checkpoint_files) > 0:
model_checkpoint = max(checkpoint_files)
trained_model_epoch = int(model_checkpoint[-5:-3])
checkpoint_complete_path = os.path.join(output_folder,
model_checkpoint)
model.load_weights(checkpoint_complete_path)
logger.info(
"Model weights have been recovered from %s",
checkpoint_complete_path,
)
else:
logger.info(("No available checkpoint for this configuration. "
"The model will be trained from scratch."))
trained_model_epoch = 0
checkpoint_filename = os.path.join(output_folder,
"checkpoint-epoch-{epoch:03d}.h5")
checkpoint = callbacks.ModelCheckpoint(
checkpoint_filename,
monitor="val_loss",
verbose=0,
save_best_only=True,
save_weights_only=False,
mode="auto",
period=1,
)
terminate_on_nan = callbacks.TerminateOnNaN()
earlystop = callbacks.EarlyStopping(monitor="val_loss",
patience=10,
verbose=1,
mode="max")
csv_logger = callbacks.CSVLogger(os.path.join(output_folder,
"training_metrics.csv"),
append=True)
hist = model.fit_generator(
train_generator,
epochs=nb_epochs,
initial_epoch=trained_model_epoch,
steps_per_epoch=steps,
validation_data=validation_generator,
validation_steps=val_steps,
callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger],
)
ref_metric = max(hist.history.get("val_acc", [np.nan]))
return {
"model": model,
"val_acc": ref_metric,
"batch_size": batch_size,
"network": network,
"dropout": dropout,
"learning_rate": learning_rate,
"learning_rate_decay": learning_rate_decay,
}
def resume():
print('-' * 30)
print('Loading and preprocessing train data...')
print('-' * 30)
imgs_train, imgs_gtruth_train = load_train_data()
print('-' * 30)
print('Loading and preprocessing validation data...')
print('-' * 30)
imgs_val, imgs_gtruth_val = load_validatation_data()
print('-' * 30)
print('Creating and compiling model...')
print('-' * 30)
if unet_model_type == 'default':
model = get_unet_default()
elif unet_model_type == 'reduced':
model = get_unet_reduced()
elif unet_model_type == 'extended':
model = get_unet_extended()
checkpoint_filepath_best = 'outputs/' + 'best_4classes_32_reduced_tuned_8915.h5'
print(checkpoint_filepath_best)
model.load_weights(checkpoint_filepath_best)
model.summary()
print('-' * 30)
print('Fitting model...')
print('-' * 30)
#============================================================================
print('training starting..')
log_filename = 'outputs/' + image_type + '_model_train.csv'
#Callback that streams epoch results to a csv file.
csv_log = callbacks.CSVLogger(log_filename, separator=',', append=True)
early_stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=10,
verbose=0,
mode='min')
#checkpoint_filepath = 'outputs/' + image_type +"_best_weight_model_{epoch:03d}_{val_loss:.4f}.hdf5"
checkpoint_filepath = 'outputs/' + 'weights.h5'
checkpoint = callbacks.ModelCheckpoint(checkpoint_filepath,
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min')
#callbacks_list = [csv_log, checkpoint]
callbacks_list = [csv_log, early_stopping, checkpoint]
#============================================================================
hist = model.fit(
imgs_train,
imgs_gtruth_train,
batch_size=batch_size,
nb_epoch=nb_epochs,
verbose=1,
validation_data=(imgs_val, imgs_gtruth_val),
shuffle=True,
callbacks=callbacks_list) # validation_split=0.2,
model_name = 'outputs/' + image_type + '_model_last'
model.save(model_name) # creates a HDF5 file 'my_model.h5'
def keras_mlp1(train2, y, test2, v, z):
cname = sys._getframe().f_code.co_name
v[cname], z[cname] = 0, 0
from keras import callbacks
from keras import layers
from keras import models
from keras import optimizers
from keras.wrappers.scikit_learn import KerasRegressor
scores = list()
scaler = preprocessing.RobustScaler()
train3 = scaler.fit_transform(train2)
test3 = scaler.transform(test2)
input_dims = train3.shape[1]
def build_model():
input_ = layers.Input(shape=(input_dims,))
model = layers.Dense(256,
kernel_initializer='Orthogonal',
activation=layers.advanced_activations.PReLU())(input_)
model = layers.BatchNormalization()(model)
#model = layers.Dropout(0.7)(model)
model = layers.Dense(64,
kernel_initializer='Orthogonal',
activation=layers.advanced_activations.PReLU())(model)
model = layers.BatchNormalization()(model)
#model = layers.Dropout(0.9)(model)
model = layers.Dense(16,
kernel_initializer='Orthogonal',
activation=layers.advanced_activations.PReLU())(model)
model = layers.BatchNormalization()(model)
model = layers.Dense(1,
activation='sigmoid')(model)
model = models.Model(input_, model)
model.compile(loss = 'binary_crossentropy',
#optimizer = optimizers.Nadam(),
optimizer = optimizers.SGD(),
metrics = ['binary_accuracy'])
#print(model.summary(line_length=120))
return model
np.random.seed(1234)
est = KerasRegressor(build_fn=build_model,
nb_epoch=10000,
batch_size=256,
#verbose=2
)
build_model().summary(line_length=120)
model_path = '../data/working/' + cname + '_keras_model.h5'
kcb = [
callbacks.EarlyStopping(
monitor='val_loss',
patience=20
#verbose=1
),
callbacks.ModelCheckpoint(
model_path,
monitor='val_loss',
save_best_only=True,
save_weights_only=True,
verbose=0
),
callbacks.ReduceLROnPlateau(
monitor='val_loss',
min_lr=1e-7,
factor=0.2,
verbose=1
)
]
num_splits = 9
ss = model_selection.ShuffleSplit(n_splits=num_splits, random_state=11)
for n, (itrain, ival) in enumerate(ss.split(train3, y)):
xtrain, xval = train3[itrain], train3[ival]
ytrain, yval = y[itrain], y[ival]
est.fit(
xtrain, ytrain,
epochs=10000,
validation_data=(xval, yval),
verbose=0,
callbacks=kcb,
shuffle=True
)
est.model.load_weights(model_path)
p = est.predict(xval)
v.loc[ival, cname] += pconvert(p)
score = metrics.log_loss(y[ival], p)
print(cname, 'fold %d: '%(n+1), score, now())
scores.append(score)
z[cname] += pconvert(est.predict(test3))
os.remove(model_path)
cv=np.array(scores)
print(cv, cv.mean(), cv.std())
z[cname] /= num_splits
def train_network(self, parameters, epochs=2, load=False):
batch_size = 128
x_train, one_hot_y_train, x_val, one_hot_y_val = self.dataset
no_of_classes = one_hot_y_train.shape[1]
#if ('embed_size' in parameters and parameters['embed_size']) and \
# ('encoder' in parameters and parameters['encoder']):
# del parameters['encoder']
if 'encoder' in parameters and not self.is_train_autoencoder:
del parameters['encoder']
print(
f"\ntrain_network(parameters={parameters}, epochs={epochs}, load={load}), no_of_classes={no_of_classes}"
)
# == formatting data ==
# trim
x_train_ = x_train[:int(np.ceil(len(x_train) * self.percent))]
y_train_ = one_hot_y_train[:int(np.ceil(len(x_train) * self.percent))]
x_val_ = x_val[:int(np.ceil(len(x_val) * self.percent))]
y_val_ = one_hot_y_val[:int(np.ceil(len(x_val) * self.percent))]
## Training Data loaded with shape: (819200, 512) and labels with shape - (819200, 2)
## Validation Data loaded with shape: (102400, 512) and labels with shape - (102400, 2)
# load existing model if exists
model_dir = self.model_dir(None, params=parameters)
#log_dir = os.path.join(model_dir, 'fit')
log_dir = os.path.join(os.path.abspath('./output'), 'fit')
parameters['accuracy'] = 0.
parameters['loss'] = np.inf
try:
if load and os.path.exists(self.model_dir(
'.h5', params=parameters)): # load old model
print('loading old model...', end=' ')
start_time = time.time()
model = models.load_model(
self.model_dir('.h5', params=parameters))
print('old model loaded in {:.2f}s.'.format(time.time() -
start_time))
else:
# default: build new model
if self.is_train_autoencoder:
optimizer = keras.optimizers.adadelta(learning_rate=1.0,
rho=0.95)
loss_func = keras.losses.mse
encoder, decoder, autoencoder = build_autoencoder(
parameters,
input_shape=(self.block_size, ),
gpus=self.gpus,
optimizer=optimizer,
loss=loss_func)
model = autoencoder
else:
# manually defining optimizer and loss
optimizer = keras.optimizers.rmsprop(lr=0.0005, rho=0.9)
loss_func = keras.losses.categorical_crossentropy
model = build_model(parameters,
no_of_classes=no_of_classes,
input_length=self.block_size,
gpus=self.gpus,
optimizer=optimizer,
loss=loss_func)
if self.is_train_autoencoder:
y_train_ = x_train_
y_val_ = x_val_
print(
f"Model path: \"{self.model_dir('.h5', params=parameters)}\"")
monitor = 'val_loss' if self.is_train_autoencoder else 'val_acc'
callbacks_list = [
callbacks.EarlyStopping(monitor=monitor,
patience=50,
restore_best_weights=True,
min_delta=0.01),
# saving model with exact name
callbacks.ModelCheckpoint(self.model_dir('.h5'),
monitor=monitor),
callbacks.CSVLogger(filename=(self.model_dir('.log')),
append=True),
# saving model hparam_str in name
callbacks.ModelCheckpoint(self.model_dir('.h5',
params=parameters),
monitor=monitor),
callbacks.CSVLogger(filename=(self.model_dir(
'.log', params=parameters)),
append=True),
]
if epochs > 2:
# tensorboard logging
callbacks_list.append(
callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1,
embeddings_freq=0))
#callbacks_list.append(callbacks.TensorBoard(log_dir=log_dir))
os.makedirs(model_dir, exist_ok=True)
model.summary()
history = model.fit(
x=x_train_,
y=y_train_,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_val_, y_val_),
verbose=self.verbose,
callbacks=callbacks_list,
)
try:
split_model_path = os.path.split(self.model_name)
size = len(split_model_path)
keras.utils.plot_model(
model,
os.path.join(self.model_dir(None, params=parameters),
split_model_path[size - 1] + '.png'),
show_shapes=True)
except Exception as e:
print(f"Couln't plot diagram: {e}")
if 'val_loss' in history.history:
parameters['loss'] = max(history.history['val_loss'])
if 'val_acc' in history.history:
parameters['accuracy'] = max(history.history['val_acc'])
print()
self.df.loc[len(self.df)] = parameters
backend.clear_session()
except ValueError as ve:
print('!!ERROR: {0}'.format(ve))
# raise exception if not whitelisted
whitelisted_exceptions = [
'Negative dimension size caused by subtracting',
'Error when checking target: expected ', ''
]
if not list(filter(str(ve).__contains__, whitelisted_exceptions)):
raise ve
print("Loss: {}".format(parameters['loss']))
print("Accuracy: {:.2%}".format(parameters['accuracy']))
return parameters['loss']
def baseline(self,
dataset,
input_dim=784,
output_dim=10,
lr=0.001,
epochs=100,
batch_size=32,
random_seed=29,
mute=False,
test=True,
noVal=False,
upload=False,
dropout=True,
class_curve=False):
#set up random seed
from numpy.random import seed
seed(random_seed)
from tensorflow import set_random_seed
set_random_seed(random_seed)
verbose = 0
if (mute):
verbose = 0
else:
verbose = 1
x_train, y_train = np.asarray(dataset.train_images), np.asarray(
dataset.train_labels)
x_val, y_val = np.asarray(dataset.validation_images), np.asarray(
dataset.validation_labels)
x_test, y_test = np.asarray(dataset.test_images), np.asarray(
dataset.test_labels)
architecture = self.architecture
if (noVal):
x_train = np.concatenate((x_train, x_val), axis=0)
y_train = np.concatenate((y_train, y_val), axis=0)
model = Sequential()
model.add(
Dense(architecture[0], activation='relu', input_dim=input_dim))
for i in range(1, len(architecture)):
model.add(Dense(architecture[i], activation='relu'))
if (dropout):
model.add(Dropout(0.5))
model.add(Dense(output_dim, activation='softmax'))
rmsprop = optimizers.RMSprop(lr=lr, rho=0.9, epsilon=None, decay=0.0)
model.compile(loss='categorical_crossentropy',
optimizer=rmsprop,
metrics=['accuracy'])
filepath = self.name + "baseline.best.hdf5"
checkpoint = callbacks.ModelCheckpoint(filepath,
monitor='val_loss',
verbose=verbose,
save_best_only=True,
mode='auto')
early_Stopping = callbacks.EarlyStopping(monitor='val_loss',
min_delta=0,
patience=20,
verbose=verbose,
mode='auto')
Callbacks = [checkpoint, early_Stopping]
#show training curve of each class
if (class_curve):
class_accu = class_accuracy()
Callbacks.append(class_accu)
if (not noVal):
model.fit(x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
callbacks=Callbacks,
validation_data=(x_val, y_val),
shuffle=True)
else:
model.fit(x_train,
y_train,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
callbacks=Callbacks,
validation_data=(x_test, y_test),
shuffle=True)
score = 0
model.load_weights(filepath)
if (test):
score = model.evaluate(x_test,
y_test,
batch_size=batch_size,
verbose=verbose)
else:
score = model.evaluate(x_test,
y_test,
batch_size=batch_size,
verbose=verbose)
if (upload):
weights = []
bias = []
for i in range(0, len(model.layers)):
parameters = model.layers[i].get_weights()
#dropout layers doesn't have weight
if (not parameters):
continue
weights.append(copy.deepcopy(parameters[0]))
bias.append(copy.deepcopy(parameters[1]))
self.weights = weights
self.bias = bias
return score
