def run(_run, image_shape, train_info, data_dir, train_pairs, valid_pairs, train_shuffle, valid_shuffle, subdirectories, architecture, weights, batch_size, last_base_layer, pooling, device, opt_params, dropout_rate, resuming_ckpt, ckpt, steps_per_epoch, epochs, validation_steps, workers, use_multiprocessing, initial_epoch, early_stop_patience, use_gram_matrix, dense_layers, limb_weights, trainable_limbs, outputs_meta): report_dir = _run.observers[0].dir print('reading train-info...') outputs, name_map = load_multiple_outputs(train_info, outputs_meta, encode='sparse') g = ImageDataGenerator( horizontal_flip=True, vertical_flip=True, zoom_range=.2, rotation_range=.2, height_shift_range=.2, width_shift_range=.2, fill_mode='reflect', preprocessing_function=get_preprocess_fn(architecture)) print('loading train meta-data...') train_data = BalancedDirectoryPairsMultipleOutputsSequence( os.path.join(data_dir, 'train'), outputs, name_map, g, batch_size=batch_size, target_size=image_shape[:2], subdirectories=subdirectories, shuffle=train_shuffle, pairs=train_pairs) print('loading valid meta-data...') valid_data = BalancedDirectoryPairsMultipleOutputsSequence( os.path.join(data_dir, 'valid'), outputs, name_map, g, batch_size=batch_size, target_size=image_shape[:2], subdirectories=subdirectories, shuffle=valid_shuffle, pairs=valid_pairs) with tf.device(device): print('building...') model = build_siamese_model( image_shape, architecture, dropout_rate, weights, last_base_layer=last_base_layer, use_gram_matrix=use_gram_matrix, dense_layers=dense_layers, pooling=pooling, include_base_top=False, include_top=True, trainable_limbs=trainable_limbs, limb_weights=limb_weights, predictions_activation=[o['a'] for o in outputs_meta], predictions_name=[o['n'] for o in outputs_meta], classes=[o['u'] for o in outputs_meta], embedding_units=[o['e'] for o in outputs_meta], joints=[o['j'] for o in outputs_meta]) print('siamese model summary:') model.summary() if resuming_ckpt: print('loading weights...') model.load_weights(resuming_ckpt) model.compile(optimizer=optimizers.Adam(**opt_params), loss=dict((o['n'] + '_binary_predictions', o['l']) for o in outputs_meta), metrics=dict((o['n'] + '_binary_predictions', o['m']) for o in outputs_meta)) print('training from epoch %i...' % initial_epoch) try: model.fit_generator( train_data, steps_per_epoch=steps_per_epoch, epochs=epochs, validation_data=valid_data, validation_steps=validation_steps, initial_epoch=initial_epoch, use_multiprocessing=use_multiprocessing, workers=workers, verbose=2, callbacks=[ callbacks.TerminateOnNaN(), callbacks.EarlyStopping(patience=early_stop_patience), callbacks.ReduceLROnPlateau(min_lr=1e-10, patience=early_stop_patience // 3), callbacks.TensorBoard(report_dir, batch_size=batch_size), callbacks.ModelCheckpoint(os.path.join(report_dir, ckpt), save_best_only=True, verbose=1), ]) except KeyboardInterrupt: print('interrupted by user') else: print('done')
def train (dataset, model, le, plot='plot.png'): """ Function Wrapper to initialize training using Keras Model. """ args = { 'dataset': dataset, 'model': model, 'le': le, 'plot': plot } # initialize the initial learning rate, batch size, and number of # epochs to train for INIT_LR = 1e-4 BS = 8 EPOCHS = 50 # grab the list of images in our dataset directory, then initialize # the list of data (i.e., images) and class images print("[INFO] loading images...") imagePaths = list(paths.list_images(args["dataset"])) data = [] labels = [] for imagePath in imagePaths: # extract the class label from the filename, load the image and # resize it to be a fixed 32x32 pixels, ignoring aspect ratio label = imagePath.split(os.path.sep)[-2] try: image = cv2.imread(imagePath) image = cv2.resize(image, (32, 32)) except Exception as e: print ("Image %s is broken. Continue.." % imagePath.split(os.path.sep)[-2:]) continue # update the data and labels lists, respectively data.append(image) labels.append(label) # convert the data into a NumPy array, then preprocess it by scaling # all pixel intensities to the range [0, 1] data = np.array(data, dtype="float") / 255.0 # encode the labels (which are currently strings) as integers and then # one-hot encode them le = LabelEncoder() labels = le.fit_transform(labels) labels = np_utils.to_categorical(labels, 2) # partition the data into training and testing splits using 75% of # the data for training and the remaining 25% for testing (trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.25, random_state=42) # construct the training image generator for data augmentation aug = ImageDataGenerator(rotation_range=20, zoom_range=0.15, width_shift_range=0.2, height_shift_range=0.2, shear_range=0.15, horizontal_flip=True, # vertical_flip=True, fill_mode="nearest" ) # initialize the optimizer and model # print("[INFO] compiling model...") # opt = Adam(lr=INIT_LR, decay=INIT_LR / EPOCHS) # model_builder = Model(width=32, height=32, depth=3, classes=len(le.classes_)) # model = model_builder.build_liveness() # VGG optimizer and model print("[INFO] compiling model...") opt = Adam(lr=INIT_LR, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=1e-6) model_builder = Model(width=32, height=32, depth=3, classes=len(le.classes_)) model = model_builder.build_VGG() nFreeze = 10 # freeze layers for layer in model.layers[:nFreeze]: layer.trainable = False # model compilation model.compile(loss="binary_crossentropy", optimizer=opt, metrics=["accuracy"]) # train the network print("[INFO] training network for {} epochs...".format(EPOCHS)) earlyStopping = callbacks.EarlyStopping(monitor='val_acc', patience=10, verbose=0, mode='auto') H = model.fit_generator( aug.flow(trainX, trainY, batch_size=BS), callbacks=[earlyStopping], validation_data=(testX, testY), steps_per_epoch=len(trainX) // BS, epochs=EPOCHS ) # evaluate the network print("[INFO] evaluating network...") predictions = model.predict(testX, batch_size=BS) print(classification_report(testY.argmax(axis=1), predictions.argmax(axis=1), target_names=le.classes_)) # save the network to disk print("[INFO] serializing network to '{}'...".format(args["model"])) modelpath = 'models/' + args['model'] model.save(modelpath) # save the label encoder to disk f = open(args["le"], "wb") f.write(pickle.dumps(le)) f.close() # plot the training loss and accuracy plt.style.use("ggplot") plt.figure() print(H.history.keys()) plt.plot(np.arange(0, EPOCHS), H.history["loss"], label="train_loss") plt.plot(np.arange(0, EPOCHS), H.history["val_loss"], label="val_loss") plt.plot(np.arange(0, EPOCHS), H.history["accuracy"], label="train_acc") plt.plot(np.arange(0, EPOCHS), H.history["val_accuracy"], label="val_acc") plt.title("Training Loss and Accuracy on Dataset") plt.xlabel("Epoch #") plt.ylabel("Loss/Accuracy") plt.legend(loc="lower left") plotpath = 'plots/' + args["plot"] plt.savefig(plotpath) plt.show()
def main(): def load_scan2(path): print("load_scan2") list1 = [] for dirName, subdirList, fileList in os.walk(path): for filename in fileList: if ".dcm" in filename.lower(): list1.append(os.path.join(dirName, filename)) return list1 bc = [] bm = [] mm = [] mc = [] img = [] final = [] final_1 = [] a = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/benign-calc' ) for i in range(0, 196): # bc.append(pdicom.read_file(a[i])) final.append(pdicom.read_file(a[i])) bclen = len(a) a = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/benign-calc' ) for i in range(0, 196): # bc.append(pdicom.read_file(a[i])) final.append(pdicom.read_file(a[i])) bclen = bclen + len(a) b = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/benign-mass' ) for i in range(0, 196): # bm.append(pdicom.read_file(b[i])) final.append(pdicom.read_file(b[i])) bmlen = len(b) b = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/benign-mass' ) for i in range(0, 196): # bm.append(pdicom.read_file(b[i])) final.append(pdicom.read_file(b[i])) bmlen = bmlen + len(b) c = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/malignant-mass' ) for i in range(0, 196): # mm.append(pdicom.read_file(c[i])) final.append(pdicom.read_file(c[i])) mmlen = len(c) # c = load_scan2('/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/malignant-mass') for i in range(0, 196): # mm.append(pdicom.read_file(c[i])) final.append(pdicom.read_file(c[i])) mmlen = mmlen + len(c) e = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/MLO/malignant-calc' ) for i in range(0, 196): # mc.append(pdicom.read_file(e[i])) final.append(pdicom.read_file(e[i])) mclen = len(e) e = load_scan2( '/home/genomics/PycharmProjects/BreastCancerPredictor/processed images-training/resized/CC/malignant-calc' ) for i in range(0, 196): # mc.append(pdicom.read_file(e[i])) final.append(pdicom.read_file(e[i])) mclen = mclen + len(e) print(len(final)) for i in range(0, len(final)): img.append(final[i].pixel_array) for i in range(0, 500): img[i] = img[i].astype(float) final_1.append(np.stack((img[i], ) * 3, axis=-1)) for i in range(500, 1000): img[i] = img[i].astype(float) final_1.append(np.stack((img[i], ) * 3, axis=-1)) for i in range(1000, 1568): img[i] = img[i].astype(float) final_1.append(np.stack((img[i], ) * 3, axis=-1)) final_1 = np.array(final_1) print(final_1[0].shape) num_classes = 4 num_of_samples = len(final_1) labels = np.ones((num_of_samples, ), dtype='int64') labels[0:392] = 0 labels[392:784] = 1 labels[784:1176] = 2 labels[1176:1568] = 3 print("labels") L = np_utils.to_categorical(labels, num_classes) final_1, L = shuffle(final_1, L, random_state=2) print("shuffle") train_x, val_x, train_y, val_y = train_test_split(final_1, L, stratify=L, test_size=0.2) print("split") print(len(train_x), len(train_y), len(val_x), len(val_y)) train_y = np.array(train_y) print("train_y") train_x = np.array(train_x) print("train_x") val_x = np.array(val_x) val_y = np.array(val_y) print("model") for i in range(0, len(train_x)): train_x[i] = train_x[i] / 255 print("normalize") model = resnet_pseudo() model.compile(loss="categorical_crossentropy", optimizer="adagrad", metrics=["accuracy"]) filename = 'model_train_new2.csv' csv_log = callbacks.CSVLogger(filename, separator=',', append=False) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='min') filepath = "/home/genomics/PycharmProjects/BreastCancerPredictor/try10.hdf5" checkpoint = callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min') callbacks_list = [csv_log, early_stopping, checkpoint] hist = model.fit(train_x, train_y, batch_size=10, epochs=4, verbose=1, validation_data=(val_x, val_y), callbacks=callbacks_list) model.save('model10.hdf5')
def calibrate(target, source, sourceIndex, predLabel, path): mmdNetLayerSizes = [25, 25] l2_penalty = 1e-2 init = lambda shape, name: initializations.normal( shape, scale=.1e-4, name=name) space_dim = target.X.shape[1] calibInput = Input(shape=(space_dim, )) block1_bn1 = BatchNormalization()(calibInput) block1_a1 = Activation('relu')(block1_bn1) block1_w1 = Dense(mmdNetLayerSizes[0], activation='linear', W_regularizer=l2(l2_penalty), init=init)(block1_a1) block1_bn2 = BatchNormalization()(block1_w1) block1_a2 = Activation('relu')(block1_bn2) block1_w2 = Dense(space_dim, activation='linear', W_regularizer=l2(l2_penalty), init=init)(block1_a2) block1_output = merge([block1_w2, calibInput], mode='sum') block2_bn1 = BatchNormalization()(block1_output) block2_a1 = Activation('relu')(block2_bn1) block2_w1 = Dense(mmdNetLayerSizes[1], activation='linear', W_regularizer=l2(l2_penalty), init=init)(block2_a1) block2_bn2 = BatchNormalization()(block2_w1) block2_a2 = Activation('relu')(block2_bn2) block2_w2 = Dense(space_dim, activation='linear', W_regularizer=l2(l2_penalty), init=init)(block2_a2) block2_output = merge([block2_w2, block1_output], mode='sum') block3_bn1 = BatchNormalization()(block2_output) block3_a1 = Activation('relu')(block3_bn1) block3_w1 = Dense(mmdNetLayerSizes[1], activation='linear', W_regularizer=l2(l2_penalty), init=init)(block3_a1) block3_bn2 = BatchNormalization()(block3_w1) block3_a2 = Activation('relu')(block3_bn2) block3_w2 = Dense(space_dim, activation='linear', W_regularizer=l2(l2_penalty), init=init)(block3_a2) block3_output = merge([block3_w2, block2_output], mode='sum') calibMMDNet = Model(input=calibInput, output=block3_output) n = target.X.shape[0] p = np.random.permutation(n) toTake = p[range(int(.2 * n))] targetXMMD = target.X[toTake] targetYMMD = target.y[toTake] targetXMMD = targetXMMD[targetYMMD != 0] targetYMMD = targetYMMD[targetYMMD != 0] targetYMMD = np.reshape(targetYMMD, (-1, 1)) n = source.X.shape[0] p = np.random.permutation(n) toTake = p[range(int(.2 * n))] sourceXMMD = source.X[toTake] sourceYMMD = predLabel[toTake] sourceXMMD = sourceXMMD[sourceYMMD != 0] sourceYMMD = sourceYMMD[sourceYMMD != 0] sourceYMMD = np.reshape(sourceYMMD, (-1, 1)) lrate = LearningRateScheduler(step_decay) optimizer = opt.rmsprop(lr=0.0) calibMMDNet.compile( optimizer=optimizer, loss=lambda y_true, y_pred: cf.MMD( block3_output, targetXMMD, MMDTargetValidation_split=0.1). KerasCost(y_true, y_pred)) sourceLabels = np.zeros(sourceXMMD.shape[0]) calibMMDNet.fit(sourceXMMD, sourceLabels, nb_epoch=500, batch_size=1000, validation_split=0.1, verbose=0, callbacks=[ lrate, mn.monitorMMD(sourceXMMD, sourceYMMD, targetXMMD, targetYMMD, calibMMDNet.predict), cb.EarlyStopping(monitor='val_loss', patience=20, mode='auto') ]) plt.close('all') calibMMDNet.save_weights( os.path.join(io.DeepLearningRoot(), path + '/ResNet' + str(sourceIndex) + '.h5')) calibrateSource = Sample(calibMMDNet.predict(source.X), source.y) calibMMDNet = None return calibrateSource
# model.add(PReLU()) # model.add(BatchNormalization()) # model.add(Dropout(0.2)) # model.add(Dense(3, init='he_normal', activation='softmax')) # adam = optimizers.Adam(lr=1.0) # model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # return(model) # y_arr = pd.DataFrame.as_matrix(pd.DataFrame(y)).ravel() # y_arr = np.array(y) # fix random seed for reproducibility seed = 7 earlyStopping = callbacks.EarlyStopping(monitor='loss', patience=1, verbose=0, mode='auto') model = KerasClassifier(build_fn=create_model, epochs=40, batch_size=50, verbose=1) # evaluate using 10-fold cross validation kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed) results = model_selection.cross_val_score( model, x_train, encoded_Y, cv=kfold, scoring='neg_log_loss', fit_params={'callbacks': [earlyStopping]})
import numpy as np # SVG 표시에 필요한 라이브러리 임포트 from IPython.display import SVG from keras.utils.vis_utils import model_to_dot # 윈도우의 경우 다음을 추가 import os os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/' # 임의의 숫자로 더미 데이터를 준비 x_train = np.random.random((100, 6, 6, 1)) y_train = keras.utils.to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) x_test = np.random.random((20, 6, 6, 1)) y_test = keras.utils.to_categorical(np.random.randint(10, size=(20, 1)), num_classes=10) # 합성곱 신경망 모델 작성 model = Sequential() model.add(Conv2D(filters=3, kernel_size=(3, 3), input_shape=(6, 6, 1), kernel_initializer='lecun_uniform', name='Conv2D_1')) model.add(Flatten(name='Flatten_1')) model.add(Dense(units=10, activation='softmax', name='Dense_1')) # 시퀀스 출력 SVG(model_to_dot(model, show_shapes=True).create(prog='dot', format='svg')) # 조기 종료 설정 earlyStopping = callbacks.EarlyStopping(monitor='val_loss', patience=5) # 모델 컴파일 model.compile(loss='mean_squared_error', optimizer='sgd') model.fit(x_train, y_train, batch_size=32, epochs=10, callbacks=[earlyStopping], validation_split=0.2)
def trainDAE(target, dataPath, refSampleInd, trainIndex, relevantMarkers, mode, keepProb, denoise, loadModel, path): sourceX = [] for i in np.arange(trainIndex.size - 1): sourceIndex = np.delete(trainIndex, refSampleInd)[i] source = dh.loadDeepCyTOFData(dataPath, sourceIndex, relevantMarkers, mode) numZerosOK = 1 toKeepS = np.sum((source.X == 0), axis=1) <= numZerosOK if i == 0: sourceX = source.X[toKeepS] else: sourceX = np.concatenate([sourceX, source.X[toKeepS]], axis=0) # preProcess source sourceX = np.log(1 + np.abs(sourceX)) numZerosOK = 1 toKeepT = np.sum((target.X == 0), axis=1) <= numZerosOK inputDim = target.X.shape[1] ae_encodingDim = 25 l2_penalty_ae = 1e-2 if denoise: if loadModel: from keras.models import load_model autoencoder = load_model( os.path.join(io.DeepLearningRoot(), 'savemodels/' + path + '/denoisedAE.h5')) else: # train de-noising auto encoder and save it. trainTarget_ae = np.concatenate([sourceX, target.X[toKeepT]], axis=0) trainData_ae = trainTarget_ae * np.random.binomial( n=1, p=keepProb, size=trainTarget_ae.shape) input_cell = Input(shape=(inputDim, )) encoded = Dense(ae_encodingDim, activation='relu', W_regularizer=l2(l2_penalty_ae))(input_cell) encoded1 = Dense(ae_encodingDim, activation='relu', W_regularizer=l2(l2_penalty_ae))(encoded) decoded = Dense(inputDim, activation='linear', W_regularizer=l2(l2_penalty_ae))(encoded1) autoencoder = Model(input=input_cell, output=decoded) autoencoder.compile(optimizer='rmsprop', loss='mse') autoencoder.fit(trainData_ae, trainTarget_ae, nb_epoch=80, batch_size=128, shuffle=True, validation_split=0.1, verbose=0, callbacks=[ mn.monitor(), cb.EarlyStopping(monitor='val_loss', patience=25, mode='auto') ]) autoencoder.save( os.path.join(io.DeepLearningRoot(), 'savemodels/' + path + '/denoisedAE.h5')) del sourceX plt.close('all') return autoencoder
def trainModel(DS, x_train, y_train, x_valid, y_valid, x_test, y_test): '''Build and Train (pretrained) ResNet50 Model''' #Prepare Dataset for training x_train, y_train, x_valid, y_valid, x_test, y_test = prepareDataset( x_train, y_train, x_valid, y_valid, x_test, y_test) # Define the parameters for ResNet50 model. IMG_HEIGHT = 32 IMG_WIDTH = 32 IMG_DEPTH = 3 NB_EPOCHS = 20 no_of_class = noClass(DS) print('Parameters are defined') #Base Model base_model = ResNet50(include_top=False, pooling='avg', weights='imagenet', input_shape=(IMG_HEIGHT, IMG_WIDTH, IMG_DEPTH)) print('Base Model is created') base_model.summary() # Adding Dense Layers x = base_model.output # Adding a fully-connected layer x = Dense(1024, activation='relu')(x) # Addinf a softmax layer -- no of class predictions = Dense(no_of_class, activation='softmax')(x) # # Create new model model = Model(inputs=base_model.input, outputs=predictions) # Conv & pooling layers are not trainable for layer in base_model.layers: layer.trainable = False print('Model is created') # Compile the model. model.compile(loss='categorical_crossentropy', optimizer=optimizers.Adam(), metrics=['acc']) print('Model is compiled') from keras import callbacks #Define callbacks reduce_learning = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2, patience=2, verbose=1, mode='auto', min_delta=0.0001, cooldown=2, min_lr=0) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=7, verbose=1, mode='auto') callbacks = [reduce_learning, early_stopping] print('Call backs are defined') print('\n\n>>>Model Trainig starts now') # Train the the model history = model.fit(x_train, y_train, epochs=NB_EPOCHS, validation_data=(x_valid, y_valid), callbacks=callbacks) acc = history.history['acc'] val_acc = history.history['val_acc'] loss = history.history['loss'] val_loss = history.history['val_loss'] print('') print(f'Model training accuracy: {acc[-1]}') print(f'Model validation accuracy: {val_loss[-1]}') print(f'Model training loss: {acc[-1]}') print(f'Model validation loss: {val_loss[-1]}') test_acc, test_precision, test_recall, test_fscore, hamming_loss = modelPerformance( model, x_test, y_test) results = {'train':[acc[-1], val_acc[-1], loss[-1], val_loss[-1]],\ 'test':[test_acc, test_precision, test_recall, test_fscore, hamming_loss]} return model, history, results
def train(self, data=None, trgt=None, ifold=0, hidden_neurons=None, layer=1): # Change elements equal to zero to one if hidden_neurons is None: hidden_neurons = [1] for i in range(len(hidden_neurons)): if hidden_neurons[i] == 0: hidden_neurons[i] = 1 if (layer <= 0) or (layer > len(hidden_neurons)): print("[-] Error: The parameter layer must be greater than zero and less " \ "or equal to the length of list hidden_neurons") return -1 neurons_str = self.get_neurons_str(data, hidden_neurons[:layer]) model_str = os.path.join( self.save_path, self.prefix_str + "_{}_neurons".format(neurons_str)) trgt_sparse = np_utils.to_categorical(trgt.astype(int)) file_name = '%s_fold_%i_model.h5' % (model_str, ifold) if os.path.exists(file_name): if self.verbose: print('File %s exists' % file_name) # load model file_name = '%s_fold_%i_model.h5' % (model_str, ifold) classifier = load_model( file_name, custom_objects={ '%s' % self.parameters["HyperParameters"]["loss"]: self.lossFunction }) file_name = '%s_fold_%i_trn_desc.jbl' % (model_str, ifold) trn_desc = joblib.load(file_name) return ifold, classifier, trn_desc train_id, test_id = self.CVO[ifold] best_init = 0 best_loss = 999 classifier = [] trn_desc = {} norm_data = self.normalize_data(data, ifold) for i_init in range(self.parameters["HyperParameters"]["n_inits"]): print( 'Neural Network - Layer: %i - Topology: %s - Fold %i of %i Folds - Init %i of %i Inits' % (layer, neurons_str, ifold + 1, self.parameters["HyperParameters"]["n_folds"], i_init + 1, self.parameters["HyperParameters"]["n_inits"])) model = Sequential() for ilayer in range(layer): if ilayer == 1: #if bool(self.parameters["HyperParameters"]["dropout"]): # model.add(Dropout(int(self.parameters["HyperParameters"]["dropout_parameter"]))) if self.parameters["HyperParameters"][ "regularization"] == "l1": model.add( Dense(units=hidden_neurons[ilayer], input_dim=data.shape[1], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"], kernel_regularizer=regularizers.l1( self.parameters["HyperParameters"] ["regularization_parameter"]))) elif self.parameters["HyperParameters"][ "regularization"] == "l2": model.add( Dense(hidden_neurons[ilayer], input_dim=data.shape[1], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"], kernel_regularizer=regularizers.l2( self.parameters["HyperParameters"] ["regularization_parameter"]))) else: model.add( Dense( hidden_neurons[ilayer], input_dim=data.shape[1], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"])) else: #if bool(self.parameters["HyperParameters"]["dropout"]): # model.add(Dropout(int(self.parameters["HyperParameters"]["dropout_parameter"]))) if self.parameters["HyperParameters"][ "regularization"] == "l1": model.add( Dense(units=hidden_neurons[ilayer], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"], kernel_regularizer=regularizers.l1( self.parameters["HyperParameters"] ["regularization_parameter"]))) elif self.parameters["HyperParameters"][ "regularization"] == "l2": model.add( Dense(hidden_neurons[ilayer], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"], kernel_regularizer=regularizers.l2( self.parameters["HyperParameters"] ["regularization_parameter"]))) else: model.add( Dense( hidden_neurons[ilayer], kernel_initializer=self.parameters[ "HyperParameters"]["kernel_initializer"])) model.add( Activation(self.parameters["HyperParameters"] ["hidden_activation_function"])) # Add Output Layer model.add( Dense(units=trgt_sparse.shape[1], kernel_initializer=self.parameters["HyperParameters"] ["kernel_initializer"])) model.add( Activation(self.parameters["HyperParameters"] ["classifier_output_activation_function"])) model.compile( loss=self.lossFunction, optimizer=self.optmizer, metrics=self.parameters["HyperParameters"]["metrics"]) # Train model earlyStopping = callbacks.EarlyStopping( monitor=self.parameters["callbacks"]["EarlyStopping"] ["monitor"], patience=self.parameters["callbacks"]["EarlyStopping"] ["patience"], verbose=self.verbose, mode='auto') class_weights = getGradientWeights(trgt[train_id]) init_trn_desc = model.fit( norm_data[train_id], trgt_sparse[train_id], epochs=self.parameters["HyperParameters"]["n_epochs"], batch_size=self.parameters["HyperParameters"]["batch_size"], callbacks=[earlyStopping], verbose=self.verbose, validation_data=(norm_data[test_id], trgt_sparse[test_id]), shuffle=True, class_weight=class_weights) if np.min(init_trn_desc.history['val_loss']) < best_loss: best_init = i_init trn_desc['best_init'] = best_init best_loss = np.min(init_trn_desc.history['val_loss']) classifier = model trn_desc['epochs'] = init_trn_desc.epoch for imetric in range( len(self.parameters["HyperParameters"]["metrics"])): if self.parameters["HyperParameters"]["metrics"][ imetric] == 'accuracy': metric = 'acc' else: metric = self.parameters["HyperParameters"]["metrics"][ imetric] trn_desc[metric] = init_trn_desc.history[metric] trn_desc['val_' + metric] = init_trn_desc.history['val_' + metric] trn_desc['loss'] = init_trn_desc.history['loss'] trn_desc['val_loss'] = init_trn_desc.history['val_loss'] # save model file_name = '%s_fold_%i_model.h5' % (model_str, ifold) classifier.save(file_name) file_name = '%s_fold_%i_trn_desc.jbl' % (model_str, ifold) joblib.dump([trn_desc], file_name, compress=9) return ifold, classifier, trn_desc
def train_Inception(train_df, test_df, label_col, batch, weights, mode): #Top 3 categorical accuracy method def top_3_categorical_accuracy(y_true, y_pred): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=3) #Top 2 categorical accuracy method def top_2_categorical_accuracy(y_true, y_pred): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=2) #Adam optimizer #Default: lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False adam = optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) #Check if binary or multiclass classification and sets parameters if mode == 0: output = 1 final_activation = 'sigmoid' optimizer_mode = adam loss_mode = 'binary_crossentropy' metrics_mode = ['binary_accuracy'] flow_mode = "binary" classes = [-1, 1] else: output = 7 final_activation = 'softmax' optimizer_mode = adam loss_mode = 'categorical_crossentropy' metrics_mode = [ 'categorical_accuracy', top_2_categorical_accuracy, top_3_categorical_accuracy ] flow_mode = "categorical" classes = [-1, 0, 1, 2, 3, 4, 5] #Create a base model using the built-in Inceptionv3 model in Keras base_model = InceptionV3(weights='imagenet', include_top=False, input_shape=(299, 299, 3), pooling='avg') #Classifier for the model x = base_model.output res_predictions = Dense(units=output, activation=final_activation)(x) #Create a functional API model using Inception v3 and a new classifier transfer = Model(inputs=base_model.input, outputs=res_predictions) #Optional freezing of layers if not enough GPU memory #for layer in transfer.layers[:52]: #Freeze block 1 #layer.trainable = False # Compile model # Optimiser parameter: adam, Loss function: Binary Crossentropy, Performance metric: Binary Accuracy transfer.compile(optimizer=optimizer_mode, loss=loss_mode, metrics=metrics_mode) #Realtime image augmentation #Rescale: rescales RGB values to 0-1 #, rotation_range = 40, width_shift_range=0.2, height_shift_range=0.2, fill_mode='nearest' train_datagen = ImageDataGenerator(preprocessing_function=preprocess_input, validation_split=0.25) test_datagen = ImageDataGenerator( preprocessing_function=preprocess_input ) #We don't modify the testing images as we are only doing predictions #Takes dataframe input and images directory and generate batches of augmented data train_generator = train_datagen.flow_from_dataframe(dataframe=train_df, directory=train_dest, x_col="file_name", y_col=label_col, has_ext=False, batch_size=batch, seed=42, shuffle=True, class_mode=flow_mode, target_size=(299, 299), subset="training", classes=classes) valid_generator = train_datagen.flow_from_dataframe(dataframe=train_df, directory=train_dest, x_col="file_name", y_col=label_col, has_ext=False, batch_size=batch, seed=42, shuffle=True, class_mode=flow_mode, target_size=(299, 299), subset="validation", classes=classes) test_generator = test_datagen.flow_from_dataframe( dataframe=test_df, directory=test_dest, x_col="file_name", y_col=None, has_ext=False, batch_size=1, seed=42, shuffle=False, class_mode=None, target_size=(299, 299)) #Returns images test_index = test_datagen.flow_from_dataframe( dataframe=test_df, directory=test_dest, x_col="file_name", y_col=label_col, has_ext=False, batch_size=1, seed=42, shuffle=False, class_mode=flow_mode, target_size=(299, 299)) #Returns images #Obtain steps required for training, evaluating and testing step_size_train = train_generator.n // train_generator.batch_size step_size_valid = valid_generator.n // valid_generator.batch_size step_size_test = test_generator.n // test_generator.batch_size #Callbacks csv_logger = callbacks.CSVLogger('./logs/' + label_col + '_training_log.csv', separator=',', append=False) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto') # Checks how to set class_weight parameter in fit_generator based on user input if weights == 'auto': index = train_generator.class_indices calc_class = class_weight.compute_class_weight( 'balanced', np.unique(train_generator.classes), train_generator.classes) class_weights = dict(zip(index.values(), calc_class)) elif weights == 'nan_suppress': index = train_generator.class_indices calc_class = class_weight.compute_class_weight( 'balanced', np.unique(train_generator.classes), train_generator.classes) calc_class[0] = 0. class_weights = dict(zip(index.values(), calc_class)) elif weights == 'equal': if mode == 1: class_weights = {0: 1., 1: 1., 2: 1., 3: 1., 4: 1., 5: 1., 6: 1.} else: class_weights = {0: 1., 1: 1.} else: class_weights = weights #Train model on generated data via batches train_history = transfer.fit_generator( train_generator, epochs=50, class_weight=class_weights, steps_per_epoch=step_size_train, validation_data=valid_generator, validation_steps=step_size_valid, verbose=1, callbacks=[csv_logger, early_stopping], workers=10) # Evaluate model and assign loss and accuracy scores score = transfer.evaluate_generator(generator=valid_generator, steps=step_size_valid, verbose=1) # Predict output test_generator.reset() prediction = transfer.predict_generator(test_generator, steps=step_size_test, verbose=1) #Returns numpy array #Obtain predicted labels in an array through binary threshold or argmax if mode == 0: predicted_class_indices = [1 if x >= 0.5 else 0 for x in prediction] else: predicted_class_indices = np.argmax(prediction, axis=1) #argmax across rows #Map predicted labels to their unique filenames labels = ( train_generator.class_indices ) #Get the class label indices (so we know what 0 and 1 refers to) labels = dict((v, k) for k, v in labels.items()) #Reverse indices predictions = [labels[k] for k in predicted_class_indices ] #Get all predictions (class) from dict filenames = test_generator.filenames #Get all filenames of predictions results = pd.DataFrame({ "Filename": filenames, "Predictions": predictions }) #Save filename and predictions to a dataframe return results, train_history, score
def createEarlyStopping(): return callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=0, mode='auto')
def train_CNN(train_df, test_df, label_col, batch, weights, mode): #Top 3 categorical accuracy method def top_3_categorical_accuracy(y_true, y_pred): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=3) #Top 2 categorical accuracy method def top_2_categorical_accuracy(y_true, y_pred): return metrics.top_k_categorical_accuracy(y_true, y_pred, k=2) #Adam optimizer #Default: lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False adam = optimizers.Adam(lr=0.00001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False) #Check if binary or multiclass classification and sets parameters if mode == 0: output = 1 #sigmoid, softmax, elu, softplus, relu, tanh final_activation = 'sigmoid' optimizer_mode = adam #binary_crossentropy, mean_squared_error, hinge, kullback_leibler_divergence, poisson loss_mode = 'binary_crossentropy' metrics_mode = ['binary_accuracy'] flow_mode = "binary" classes = [-1, 1] else: output = 7 final_activation = 'softmax' optimizer_mode = adam loss_mode = 'categorical_crossentropy' metrics_mode = [ 'categorical_accuracy', top_2_categorical_accuracy, top_3_categorical_accuracy ] flow_mode = "categorical" classes = [-1, 0, 1, 2, 3, 4, 5] classifier = Sequential( ) #Create Sequential object which allows layers to be used # Convolution layer with Non Linearity # To extract features from input # Filters: 32, Filter shape: 3X3, Input shape: 256X256, RGB(3), Activation function: Rectifier Linear Unit classifier.add( Conv2D(32, (3, 3), input_shape=(256, 256, 3), activation='relu')) # Pooling layer using Max Pooling # To reduce the size of images and total number of nodes # Pool size: 2X2 classifier.add(MaxPooling2D(pool_size=(2, 2))) # Dropout for regularisation #classifier.add(Dropout(0.2)) #More convolution and pooling layers classifier.add(Conv2D(32, (3, 3), activation='relu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) #classifier.add(Dropout(0.2)) classifier.add(Conv2D(64, (3, 3), activation='relu')) classifier.add(MaxPooling2D(pool_size=(2, 2))) #classifier.add(Dropout(0.2)) # Flattening # Convert 3D feature map to 1D feature vector classifier.add(Flatten()) # Fully Connected/ Hidden layer # A fully connected layer # Nodes: 128, Activation function: Rectifier Linear Unit classifier.add(Dense(units=128, activation='relu')) #classifier.add(Dropout(0.2)) # Output layer # One node for binary output # Activation function: Sigmoid classifier.add(Dense(units=output, activation=final_activation)) # Compile CNN # Optimiser parameter: adam, Loss function: Binary Crossentropy, Performance metric: Binary Accuracy classifier.compile(optimizer=optimizer_mode, loss=loss_mode, metrics=metrics_mode) #Realtime image augmentation #Rescale: rescales RGB values to 0-1 #, rotation_range = 40, width_shift_range=0.2, height_shift_range=0.2, fill_mode='nearest' train_datagen = ImageDataGenerator(rescale=1. / 255, validation_split=0.25, shear_range=0.2, zoom_range=0.2, horizontal_flip=True) test_datagen = ImageDataGenerator( rescale=1. / 255 ) #We don't modify the testing images as we are only doing predictions #Takes dataframe input and images directory and generate batches of augmented data train_generator = train_datagen.flow_from_dataframe(dataframe=train_df, directory=train_dest, x_col="file_name", y_col=label_col, has_ext=False, batch_size=batch, seed=42, shuffle=True, class_mode=flow_mode, target_size=(256, 256), subset="training", classes=classes) valid_generator = train_datagen.flow_from_dataframe(dataframe=train_df, directory=train_dest, x_col="file_name", y_col=label_col, has_ext=False, batch_size=batch, seed=42, shuffle=True, class_mode=flow_mode, target_size=(256, 256), subset="validation", classes=classes) test_generator = test_datagen.flow_from_dataframe( dataframe=test_df, directory=test_dest, x_col="file_name", y_col=None, has_ext=False, batch_size=1, seed=42, shuffle=False, class_mode=None, target_size=(256, 256)) #Returns images #test_generator = test_datagen.flow_from_directory(directory = test_dest, batch_size = 1, seed = 42, shuffle = False, class_mode = None, target_size = (256, 256)) #Returns images #Obtain steps required for training, evaluating and testing step_size_train = train_generator.n // train_generator.batch_size step_size_valid = valid_generator.n // valid_generator.batch_size step_size_test = test_generator.n // test_generator.batch_size #Callbacks tensorboard = callbacks.TensorBoard(log_dir='./logs', histogram_freq=0, write_graph=True, write_images=True) csv_logger = callbacks.CSVLogger('./logs/' + label_col + '_training_log.csv', separator=',', append=False) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='auto') # Checks how to set class_weight parameter in fit_generator based on user input if weights == 'auto': index = train_generator.class_indices calc_class = class_weight.compute_class_weight( 'balanced', np.unique(train_generator.classes), train_generator.classes) class_weights = dict(zip(index.values(), calc_class)) elif weights == 'nan_suppress': index = train_generator.class_indices calc_class = class_weight.compute_class_weight( 'balanced', np.unique(train_generator.classes), train_generator.classes) calc_class[0] = 0. class_weights = dict(zip(index.values(), calc_class)) elif weights == 'equal': if mode == 1: class_weights = {0: 1., 1: 1., 2: 1., 3: 1., 4: 1., 5: 1., 6: 1.} else: class_weights = {0: 1., 1: 1.} else: class_weights = weights #Train model on generated data via batches train_history = classifier.fit_generator( train_generator, epochs=100, class_weight=class_weights, steps_per_epoch=step_size_train, validation_data=valid_generator, validation_steps=step_size_valid, verbose=1, callbacks=[csv_logger, early_stopping], workers=10) # Save weights classifier.save_weights('CNN.h5') # Evaluate model and assign loss and accuracy scores score = classifier.evaluate_generator(generator=valid_generator, steps=step_size_valid, verbose=1) # Predict output test_generator.reset() prediction = classifier.predict_generator(test_generator, steps=step_size_test, verbose=1) #Returns numpy array #Obtain predicted labels in an array through binary threshold or argmax if mode == 0: predicted_class_indices = [1 if x >= 0.5 else 0 for x in prediction] else: predicted_class_indices = np.argmax(prediction, axis=1) #argmax across rows #Map predicted labels to their unique filenames labels = ( train_generator.class_indices ) #Get the class label indices (so we know what 0 and 1 refers to) labels = dict((v, k) for k, v in labels.items()) #Reverse indices predictions = [labels[k] for k in predicted_class_indices ] #Get all predictions (class) from dict filenames = test_generator.filenames #Get all filenames of predictions results = pd.DataFrame({ "Filename": filenames, "Predictions": predictions }) #Save filename and predictions to a dataframe return results, train_history, score
def build_LSTM_model(trainData, trainBatches, testData, testBatches, windowSize, class_count, numCalls, batch_size): # Specify number of units # https://stackoverflow.com/questions/37901047/what-is-num-units-in-tensorflow-basiclstmcell#39440218 num_units = 128 embedding_size = 256 # https://keras.io/callbacks/#earlystopping early_stop = cb.EarlyStopping(monitor='sparse_categorical_accuracy', min_delta=0.0001, patience=3) # We need to add an embedding layer because LSTM (at this moment) that the API call indices (numbers) # are of some mathematical significance. E.g., system call 2 is "closer" to system calls 3 and 4. # But system call numbers have nothing to do with their semantic meaning and relation to other # system calls. So we transform it using an embedding layer so the LSTM can figure these relationships # out for itself. # https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html # https://stackoverflow.com/questions/40695452/stateful-lstm-with-embedding-layer-shapes-dont-match api_count = numCalls + 1 # +1 because 0 is our padding number inp = Input(shape=(windowSize, )) emb = Embedding(input_dim=api_count, output_dim=256, input_length=windowSize)(inp) # https://keras.io/layers/recurrent/#lstm # model.add(LSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False)) #TODO - GPU stuffs # model.add(CuDNNLSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False)) # From malconv paper filt = Conv1D(filters=64, kernel_size=3, strides=1, use_bias=True, activation='relu', padding='valid')(emb) attn = Conv1D(filters=64, kernel_size=3, strides=1, use_bias=True, activation='sigmoid', padding='valid')(emb) gated = Multiply()([filt, attn]) drop = Dropout(0.5)(gated) feat = GlobalMaxPooling1D()(drop) dense = Dense(128, activation='relu')(feat) outp = Dense(class_count, activation='sigmoid')(dense) model = Model(inp, outp) # Which optimizer to use # https://keras.io/optimizers/ opt = optimizers.RMSprop(lr=0.01, decay=0.001) # https://keras.io/models/model/#compile model.compile( loss='sparse_categorical_crossentropy', optimizer=opt, # Metrics to print # We use sparse_categorical_accuracy as opposed to categorical_accuracy # because: https://stackoverflow.com/questions/44477489/keras-difference-between-categorical-accuracy-and-sparse-categorical-accuracy # I.e., since we don't use hot-encoding, we use sparse_categorical_accuracy metrics=['sparse_categorical_accuracy']) # https://keras.io/models/model/#fit_generator hist = model.fit_generator( # Data to train trainData, # Use multiprocessing because python Threading isn't really # threading: https://docs.python.org/3/glossary.html#term-global-interpreter-lock use_multiprocessing=True, # Number of steps per epoch (this is how we train our large # number of samples dataset without running out of memory) steps_per_epoch=trainBatches, #TODO # Number of epochs epochs=100, # Validation data (will not be trained on) validation_data=testData, validation_steps=testBatches, # Do not shuffle batches. shuffle=False, # List of callbacks to be called while training. callbacks=[early_stop]) return model, hist
authors_max_length=authors_max_length, authors_input_size=authors_input_size, authors_embedding_matrix=authors_embedding_matrix, node_2_vec_emb_size=NODE_2_VEC_EMB_SIZE, regularization=regularization, static_input_size=STATIC_INPUT_SIZE, dropout=DROPOUT, opt=opt) callbacks_list = [ callbacks.TensorBoard(log_dir=TENSORBOARD_LOGS_DIR, histogram_freq=0, embeddings_freq=1, write_graph=True, write_images=False), callbacks.EarlyStopping(monitor='acc', patience=10), callbacks.ModelCheckpoint(filepath=model_outfile, monitor='val_loss', save_best_only=True), callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2) ] x_train_input = [ x_train_abstracts_padded, x_train_titles_padded, x_train_authors_padded, x_train_static ] x_val_input = [ x_val_abstracts_padded, x_val_titles_padded, x_val_authors_padded, x_val_static ]
# 使得validation数据量大小为batch_size的整数陪 nb_val_samples = len(y_val) - len(y_val) % batch_size model = get_model(shape) print(model.summary()) # 根据validation loss保存最优模型 save_best = callbacks.ModelCheckpoint('best_model.h5', monitor='val_loss', verbose=1, save_best_only=True, mode='min') # 如果训练持续没有validation loss的提升, 提前结束训练 early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=15, verbose=0, mode='auto') tbCallback = callbacks.TensorBoard(log_dir='./Graph', write_graph=True) callbacks_list = [early_stop, save_best, tbCallback] history = model.fit_generator( batch_generator(X_train, y_train, batch_size, shape, training=True, data_dir=data_path), steps_per_epoch=samples_per_epoch, validation_steps=nb_val_samples // batch_size, validation_data=batch_generator(X_val,
#n_dim = combined_feat.dimension() #scale = pre.MinMaxScaler(feature_range=(-1, 1)) #scale.fit(np.concatenate(combined_data)) # Create chunked data. #all_data = scale.transform(np.concatenate(combined_data)) all_data = np.load('all_data.npy') n_dim = all_data.shape[1] n_chunks = args.n_chunks n_data = all_data.shape[0] n_batch = n_data // n_chunks chunked_data = [all_data[i:i + n_batch] for i in range(0, n_data, n_batch)] k_folds = KFold(n_splits=n_folds, shuffle=True) calls = [callbacks.EarlyStopping(patience=30, restore_best_weights=True)] train_scores = np.zeros(n_folds) test_scores = np.zeros(n_folds) hde_timescales = [] msm_timescales = [] for i in range(n_folds): #train_val_data = [chunked_data[j] for j in train_idx] #test_data = [chunked_data[j] for j in test_idx] print('Processing fold {}'.format(i)) train_data, test_data = train_test_split(chunked_data, test_size=0.5) hde = HDE(
model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(200, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("relu")) model.add(layers.BatchNormalization()) model.add( layers.Dense(10, kernel_initializer=inits.RandomUniform( maxval=0.5, minval=-0.5))) model.add(layers.Activation("softmax")) es = clb.EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True) model.compile(optimizer=opt.Adam(), loss="categorical_crossentropy", metrics=['acc']) model.fit(xtrain, ytrain, epochs=epochs, batch_size=100, validation_split=(1 / 6), callbacks=[es]) model_list.append(model) inputs.extend(model.inputs) outputs.extend(model.outputs)
# print("y_train shape :", y_train.shape) # print('x_val shape :', x_val.shape) # print('y_val shape :', y_val.shape) # print("x_test shape :", x_test.shape) # print("y_test shape :", y_test.shape) print("x_train shape :", x_train.shape, x_train1.shape) print("y_train shape :", y_train.shape) print('x_val shape :', x_val.shape, x_val1.shape) print('y_val shape :', y_val.shape) print("x_test shape :", x_test.shape, x_test1.shape) print("y_test shape :", y_test.shape) model = IN_net.model() # kcallback earlyStopping6 = kcallbacks.EarlyStopping(monitor='val_loss', patience=patience, verbose=1, mode='auto') saveBestModel6 = kcallbacks.ModelCheckpoint(filepath='ckpt/best.h5', monitor='val_loss', verbose=1,save_best_only=True, mode='auto') # reduce_lr = ReduceLROnPlateau(monitor='val_loss',factor=0.5, patience=20, mode='auto') print("----------------------------training-begin------------------------------") tic6 = time.clock() History = model.fit([x_train, x_train1], y_train, nb_epoch =nb_epoch, batch_size=16, callbacks=[earlyStopping6, saveBestModel6], validation_data=([x_val, x_val1], y_val)) toc6 = time.clock() # model.save("models/two_channel_IN.h5") # model.save("models/spa_network.h5") model.save("models/test3_best.h5") scores = model.evaluate([x_val, x_val1], y_val, batch_size=25) print('Test score:', scores[0])
def run(_run, image_shape, data_dir, train_shuffle, dataset_train_seed, valid_shuffle, dataset_valid_seed, classes, class_mode, class_weight, architecture, weights, batch_size, base_layers, pooling, dense_layers, predictions_activation, metrics, loss, device, opt_params, dropout_p, resuming_from_ckpt_file, steps_per_epoch, epochs, validation_steps, workers, use_multiprocessing, initial_epoch, early_stop_patience, tensorboard_tag, first_trainable_layer): report_dir = _run.observers[0].dir g = ImageDataGenerator( horizontal_flip=True, vertical_flip=True, samplewise_center=True, samplewise_std_normalization=True, zoom_range=45, rotation_range=.2, height_shift_range=.2, width_shift_range=.2, fill_mode='reflect', preprocessing_function=get_preprocess_fn(architecture)) if isinstance(classes, int): classes = sorted(os.listdir(os.path.join(data_dir, 'train')))[:classes] train_data = g.flow_from_directory(os.path.join(data_dir, 'train'), target_size=image_shape[:2], classes=classes, class_mode=class_mode, batch_size=batch_size, shuffle=train_shuffle, seed=dataset_train_seed) valid_data = g.flow_from_directory(os.path.join(data_dir, 'valid'), target_size=image_shape[:2], classes=classes, class_mode=class_mode, batch_size=batch_size, shuffle=valid_shuffle, seed=dataset_valid_seed) if class_weight == 'balanced': class_weight = get_class_weights(train_data.classes) if steps_per_epoch is None: steps_per_epoch = ceil(train_data.n / batch_size) if validation_steps is None: validation_steps = ceil(valid_data.n / batch_size) with tf.device(device): print('building...') model = build_gram_model(image_shape, architecture=architecture, weights=weights, dropout_p=dropout_p, classes=train_data.num_classes, base_layers=base_layers, pooling=pooling, dense_layers=dense_layers, predictions_activation=predictions_activation) model.summary() layer_names = [l.name for l in model.layers] if first_trainable_layer: if first_trainable_layer not in layer_names: raise ValueError('%s is not a layer in the model: %s' % (first_trainable_layer, layer_names)) for layer in model.layers: if layer.name == first_trainable_layer: break layer.trainable = False model.compile(optimizer=optimizers.Adam(**opt_params), metrics=metrics, loss=loss) if resuming_from_ckpt_file: print('re-loading weights...') model.load_weights(resuming_from_ckpt_file) print('training from epoch %i...' % initial_epoch) try: model.fit_generator( train_data, steps_per_epoch=steps_per_epoch, epochs=epochs, verbose=2, validation_data=valid_data, validation_steps=validation_steps, initial_epoch=initial_epoch, class_weight=class_weight, workers=workers, use_multiprocessing=use_multiprocessing, callbacks=[ # callbacks.LearningRateScheduler(lambda epoch: .5 ** (epoch // 10) * opt_params['lr']), callbacks.TerminateOnNaN(), callbacks.ReduceLROnPlateau(min_lr=1e-10, patience=int( early_stop_patience // 3)), callbacks.EarlyStopping(patience=early_stop_patience), callbacks.TensorBoard(os.path.join(report_dir, tensorboard_tag), batch_size=batch_size), callbacks.ModelCheckpoint(os.path.join( report_dir, 'weights.h5'), save_best_only=True, verbose=1), ]) except KeyboardInterrupt: print('interrupted by user') else: print('done')
else: decoder_final=Conv2D(1,(3,3),activation='sigmoid',data_format='channels_first',padding='same')(decoder_layer_2) logging.debug(str(decoder_final.shape)) autoencoder=Model(input_img, decoder_final) autoencoder.compile(optimizer=args.opt_name, loss=args.loss_name) #start training logging.info("Training with "+image_type+" images") logging.info("Current training set is made from "+str(len(train_data_list))+" files and has "+str(get_num_samples(train_data_list,image_type))+" examples") early_stopping = callbacks.EarlyStopping(monitor='val_loss', patience=3) # Well this is sorta useless here, since nb epochs is actually set to 1 in model.fit and the number of epochs here is the number of time the outer loop runs tensorboard=callbacks.TensorBoard(log_dir='../logs', histogram_freq=1, batch_size=32, write_graph=True, write_grads=True, write_images=True, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None) training_history=train_histories() num_epochs=10 patience=3 #number of epochs where we see no improvement after which we stop training last_epoch_val_loss=1000 #arbitrarily large number perc_decrease_per_epoch=10 epochwise_loss_history=[] batchwise_loss_history=[] epochwise_val_loss_history=[] batchwise_val_loss_history=[] epoch_count=0
width_shift_range=0.05, height_shift_range=0.05, shear_range=0., zoom_range=0.1, horizontal_flip=True, vertical_flip=False, dim_ordering='th') datagen.fit(X_train) if zca_flag: print 'zca whitening enabled, saving zca matrix...' datagen.principal_components.dump('zca_matrix') print 'training model...' model.fit_generator(datagen.flow(X_train, y_train, batch_size=128, shuffle=True), samples_per_epoch=len(X_train), nb_val_samples=len(X_test), class_weight={0: 1, 1: 1}, nb_epoch=200, verbose=1, validation_data=datagen.flow(X_val, y_val, shuffle=True), callbacks=[callbacks.EarlyStopping(monitor='val_acc', patience=20, verbose=1, mode='auto')]) print 'training complete' print 'evaluating model...' score = model.evaluate(X_test, y_test, batch_size=32, verbose=1) print 'test accuracy : ' + str(score[1]) print 'saving architecture and weights...' json_string = model.to_json() open('model.json', 'w').write(json_string) model.save_weights('weights.h5')
def train(data_folder, fn_sequences, batch_size, n_epochs, output_folder): with open(os.path.join(data_folder, fn_sequences), 'r') as f: seqs = f.read().split('\n') tokenizer = Tokenizer() tokenizer.fit_on_texts(seqs) sequences = tokenizer.texts_to_sequences(seqs) vocab_size = len(tokenizer.word_index) + 1 print(vocab_size) from keras.utils import to_categorical sequences = np.array(sequences) X, y = sequences[:, :-1], sequences[:, -1] y = keras.utils.to_categorical(y, num_classes=vocab_size) seq_length = X.shape[1] # define model model = Sequential() model.add(Embedding(vocab_size, 50, input_length=seq_length)) model.add(LSTM(100, return_sequences=True)) model.add(LSTM(100)) model.add(Dense(100, activation='relu')) model.add(Dense(vocab_size, activation='softmax')) print(model.summary()) model_checkpoint = callbacks.ModelCheckpoint("my_checkpoint.h5", save_best_only=True) early_stopping = callbacks.EarlyStopping(patience=50) # compile model model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) # fit model # start an Azure ML run run = Run.get_context() class LogRunMetrics(callbacks.Callback): # callback at the end of every epoch def on_epoch_end(self, epoch, log): # log a value repeated which creates a list run.log('Loss', log['loss']) # model.fit(X, y, batch_size=256, epochs=200, callbacks=[early_stopping, model_checkpoint]) history = model.fit( X, y, batch_size=batch_size, epochs=n_epochs, verbose=2, callbacks=[LogRunMetrics(), early_stopping, model_checkpoint]) # log a single value plt.figure(figsize=(6, 3)) plt.title('Goethe LM with Keras MLP ({} epochs)'.format(n_epochs), fontsize=14) plt.plot(history.history['loss'], 'r--', label='Loss', lw=4, alpha=0.5) plt.legend(fontsize=12) plt.grid(True) # log an image run.log_image('Loss', plot=plt) # create a ./outputs/model folder in the compute target # files saved in the "./outputs" folder are automatically uploaded into run history folder_model = os.path.join(output_folder, 'model_goethe') os.makedirs(folder_model, exist_ok=True) # save the tokenizer pickle.dump(tokenizer, open(os.path.join(folder_model, 'tokenizer.pkl', 'wb'))) # save the model to file # model.save('model_goethe_generator.h5') # serialize NN architecture to JSON model_json = model.to_json() # save model JSON with open(os.path.join(folder_model, 'model.json'), 'w') as f: f.write(model_json) # save model weights model.save_weights(os.path.join(folder_model, 'model_weights.h5')) print("model saved in folder " + folder_model) return model
es = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto') ''' folds = 1 rmse_avg = [] r2_avg = [] nrmse_avg = [] loss_avg = [] count = 0 for i in tqdm(range(folds)): es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=7, verbose=0, mode='auto', restore_best_weights=True) X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.2) print(X_train.shape, X_test.shape, Y_train.shape, Y_test.shape) model = Sequential() model.add( layers.Dense(256, activation='relu', input_shape=(X_train.shape[1], ))) #model.add(layers.Dropout(0.3)) model.add(layers.Dense(128, activation='relu')) model.add(layers.Dense(64, activation='relu')) #model.add(layers.Dropout(0.3)) model.add(layers.Dense(32, activation='relu')) #model.add(layers.Dropout(0.3)) model.add(layers.Dense(16, activation='relu')) model.add(layers.Dense(8, activation='relu'))
pred = self.model.predict(self.validation_data[0]) prediction = np.round(np.asarray(pred)) target = self.validation_data[1] precision = sklm.precision_score(target, prediction) recall = sklm.recall_score(target, prediction) f1_score = sklm.f1_score(target, prediction) avg_precision = sklm.average_precision_score(target,prediction,average='weighted') print("Metrics-val:", "precision=", precision, ",recall=", recall, ",f1=", f1_score, ",avg_prec=", avg_precision) metrics = Metrics(x_tr,y_tr) #------------------------------------------------------------------- # early stopping earlyStopping=CB.EarlyStopping(monitor='val_loss', patience=10, mode='auto') # start learning model.fit(x_tr, y_tr, validation_data=(x_va, y_va), epochs=epoch_count, batch_size=batch_size, #callbacks=[metrics, earlyStopping], callbacks=[earlyStopping], verbose=2) np.set_printoptions(threshold=np.inf) print("\nPrediction on validation set:") va_predictions = model.predict(x_va, batch_size, 1) #print("VA_PREDICTIONS:",va_predictions) print("\nPrediction on training set:") tr_predictions = model.predict(x_tr, batch_size,1) # print the label against the rounded prediction for easy comparison
mode='easgd', sync_every=args.sync_every, worker_optimizer=args.worker_optimizer, elastic_force=args.elastic_force/(comm.Get_size()-1), elastic_lr=args.elastic_lr, elastic_momentum=args.elastic_momentum) else: algo = Algo(args.optimizer, loss=args.loss, validate_every=validate_every, sync_every=args.sync_every, worker_optimizer=args.worker_optimizer) # Most Keras callbacks are supported callbacks = [] callbacks.append( cbks.ModelCheckpoint( '_'.join([ model_name,args.trial_name,"mpi_learn_result.h5"]), monitor='val_loss', verbose=1 ) ) if args.early_stopping is not None: callbacks.append( cbks.EarlyStopping( patience=args.early_stopping, verbose=1 ) ) # Creating the MPIManager object causes all needed worker and master nodes to be created manager = MPIManager( comm=comm, data=data, algo=algo, model_builder=model_builder, num_epochs=args.epochs, train_list=train_list, val_list=val_list, num_masters=args.masters, synchronous=args.synchronous, callbacks=callbacks, verbose=args.verbose ) # Process 0 launches the training procedure if comm.Get_rank() == 0: print (algo) t_0 = time() histories = manager.process.train() delta_t = time() - t_0 manager.free_comms()
def run_model( train_generator, validation_generator, dl_model, output_folder, instance_name, image_size, aggregate_value, nb_labels, nb_epochs, nb_training_image, nb_validation_image, batch_size, dropout, network, learning_rate, learning_rate_decay, ): """Run deep learning `dl_model` starting from training and validation data generators, depending on a range of hyperparameters Parameters ---------- train_generator : generator Training data generator validation_generator : generator Validation data generator dl_model : str Name of the addressed research problem (*e.g.* `feature_detection` or `semantic_segmentation`) output_folder : str Name of the folder where the trained model will be stored on the file system instance_name : str Name of the instance image_size : int Size of images, in pixel (height=width) aggregate_value : str Label aggregation policy (either `full` or `aggregated`) nb_labels : int Number of labels into the dataset nb_epochs : int Number of epochs during which models will be trained nb_training_image : int Number of images into the training dataset nb_validation_image : int Number of images into the validation dataset batch_size : int Number of images into each batch dropout : float Probability of keeping a neuron during dropout phase network : str Neural network architecture (*e.g.* `simple`, `vgg`, `inception`) learning_rate : float Starting learning rate learning_rate_decay : float Learning rate decay Returns ------- dict Dictionary that summarizes the instance and the corresponding model performance (measured by validation accuracy) """ if dl_model == "featdet": net = FeatureDetectionNetwork( network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network, ) loss_function = "binary_crossentropy" elif dl_model == "semseg": net = SemanticSegmentationNetwork( network_name=instance_name, image_size=image_size, nb_channels=3, nb_labels=nb_labels, architecture=network, ) loss_function = "categorical_crossentropy" else: logger.error(("Unrecognized model: %s. Please choose amongst %s", dl_model, AVAILABLE_MODELS)) sys.exit(1) model = Model(net.X, net.Y) opt = Adam(lr=learning_rate, decay=learning_rate_decay) metrics = ["acc", iou, dice_coef] model.compile(loss=loss_function, optimizer=opt, metrics=metrics) # Model training steps = max(nb_training_image // batch_size, 1) val_steps = max(nb_validation_image // batch_size, 1) checkpoint_files = [ item for item in os.listdir(output_folder) if "checkpoint-epoch" in item ] if len(checkpoint_files) > 0: model_checkpoint = max(checkpoint_files) trained_model_epoch = int(model_checkpoint[-5:-3]) checkpoint_complete_path = os.path.join(output_folder, model_checkpoint) model.load_weights(checkpoint_complete_path) logger.info( "Model weights have been recovered from %s", checkpoint_complete_path, ) else: logger.info(("No available checkpoint for this configuration. " "The model will be trained from scratch.")) trained_model_epoch = 0 checkpoint_filename = os.path.join(output_folder, "checkpoint-epoch-{epoch:03d}.h5") checkpoint = callbacks.ModelCheckpoint( checkpoint_filename, monitor="val_loss", verbose=0, save_best_only=True, save_weights_only=False, mode="auto", period=1, ) terminate_on_nan = callbacks.TerminateOnNaN() earlystop = callbacks.EarlyStopping(monitor="val_loss", patience=10, verbose=1, mode="max") csv_logger = callbacks.CSVLogger(os.path.join(output_folder, "training_metrics.csv"), append=True) hist = model.fit_generator( train_generator, epochs=nb_epochs, initial_epoch=trained_model_epoch, steps_per_epoch=steps, validation_data=validation_generator, validation_steps=val_steps, callbacks=[checkpoint, earlystop, terminate_on_nan, csv_logger], ) ref_metric = max(hist.history.get("val_acc", [np.nan])) return { "model": model, "val_acc": ref_metric, "batch_size": batch_size, "network": network, "dropout": dropout, "learning_rate": learning_rate, "learning_rate_decay": learning_rate_decay, }
def resume(): print('-' * 30) print('Loading and preprocessing train data...') print('-' * 30) imgs_train, imgs_gtruth_train = load_train_data() print('-' * 30) print('Loading and preprocessing validation data...') print('-' * 30) imgs_val, imgs_gtruth_val = load_validatation_data() print('-' * 30) print('Creating and compiling model...') print('-' * 30) if unet_model_type == 'default': model = get_unet_default() elif unet_model_type == 'reduced': model = get_unet_reduced() elif unet_model_type == 'extended': model = get_unet_extended() checkpoint_filepath_best = 'outputs/' + 'best_4classes_32_reduced_tuned_8915.h5' print(checkpoint_filepath_best) model.load_weights(checkpoint_filepath_best) model.summary() print('-' * 30) print('Fitting model...') print('-' * 30) #============================================================================ print('training starting..') log_filename = 'outputs/' + image_type + '_model_train.csv' #Callback that streams epoch results to a csv file. csv_log = callbacks.CSVLogger(log_filename, separator=',', append=True) early_stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=10, verbose=0, mode='min') #checkpoint_filepath = 'outputs/' + image_type +"_best_weight_model_{epoch:03d}_{val_loss:.4f}.hdf5" checkpoint_filepath = 'outputs/' + 'weights.h5' checkpoint = callbacks.ModelCheckpoint(checkpoint_filepath, monitor='val_loss', verbose=1, save_best_only=True, mode='min') #callbacks_list = [csv_log, checkpoint] callbacks_list = [csv_log, early_stopping, checkpoint] #============================================================================ hist = model.fit( imgs_train, imgs_gtruth_train, batch_size=batch_size, nb_epoch=nb_epochs, verbose=1, validation_data=(imgs_val, imgs_gtruth_val), shuffle=True, callbacks=callbacks_list) # validation_split=0.2, model_name = 'outputs/' + image_type + '_model_last' model.save(model_name) # creates a HDF5 file 'my_model.h5'
def keras_mlp1(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name v[cname], z[cname] = 0, 0 from keras import callbacks from keras import layers from keras import models from keras import optimizers from keras.wrappers.scikit_learn import KerasRegressor scores = list() scaler = preprocessing.RobustScaler() train3 = scaler.fit_transform(train2) test3 = scaler.transform(test2) input_dims = train3.shape[1] def build_model(): input_ = layers.Input(shape=(input_dims,)) model = layers.Dense(256, kernel_initializer='Orthogonal', activation=layers.advanced_activations.PReLU())(input_) model = layers.BatchNormalization()(model) #model = layers.Dropout(0.7)(model) model = layers.Dense(64, kernel_initializer='Orthogonal', activation=layers.advanced_activations.PReLU())(model) model = layers.BatchNormalization()(model) #model = layers.Dropout(0.9)(model) model = layers.Dense(16, kernel_initializer='Orthogonal', activation=layers.advanced_activations.PReLU())(model) model = layers.BatchNormalization()(model) model = layers.Dense(1, activation='sigmoid')(model) model = models.Model(input_, model) model.compile(loss = 'binary_crossentropy', #optimizer = optimizers.Nadam(), optimizer = optimizers.SGD(), metrics = ['binary_accuracy']) #print(model.summary(line_length=120)) return model np.random.seed(1234) est = KerasRegressor(build_fn=build_model, nb_epoch=10000, batch_size=256, #verbose=2 ) build_model().summary(line_length=120) model_path = '../data/working/' + cname + '_keras_model.h5' kcb = [ callbacks.EarlyStopping( monitor='val_loss', patience=20 #verbose=1 ), callbacks.ModelCheckpoint( model_path, monitor='val_loss', save_best_only=True, save_weights_only=True, verbose=0 ), callbacks.ReduceLROnPlateau( monitor='val_loss', min_lr=1e-7, factor=0.2, verbose=1 ) ] num_splits = 9 ss = model_selection.ShuffleSplit(n_splits=num_splits, random_state=11) for n, (itrain, ival) in enumerate(ss.split(train3, y)): xtrain, xval = train3[itrain], train3[ival] ytrain, yval = y[itrain], y[ival] est.fit( xtrain, ytrain, epochs=10000, validation_data=(xval, yval), verbose=0, callbacks=kcb, shuffle=True ) est.model.load_weights(model_path) p = est.predict(xval) v.loc[ival, cname] += pconvert(p) score = metrics.log_loss(y[ival], p) print(cname, 'fold %d: '%(n+1), score, now()) scores.append(score) z[cname] += pconvert(est.predict(test3)) os.remove(model_path) cv=np.array(scores) print(cv, cv.mean(), cv.std()) z[cname] /= num_splits
def train_network(self, parameters, epochs=2, load=False): batch_size = 128 x_train, one_hot_y_train, x_val, one_hot_y_val = self.dataset no_of_classes = one_hot_y_train.shape[1] #if ('embed_size' in parameters and parameters['embed_size']) and \ # ('encoder' in parameters and parameters['encoder']): # del parameters['encoder'] if 'encoder' in parameters and not self.is_train_autoencoder: del parameters['encoder'] print( f"\ntrain_network(parameters={parameters}, epochs={epochs}, load={load}), no_of_classes={no_of_classes}" ) # == formatting data == # trim x_train_ = x_train[:int(np.ceil(len(x_train) * self.percent))] y_train_ = one_hot_y_train[:int(np.ceil(len(x_train) * self.percent))] x_val_ = x_val[:int(np.ceil(len(x_val) * self.percent))] y_val_ = one_hot_y_val[:int(np.ceil(len(x_val) * self.percent))] ## Training Data loaded with shape: (819200, 512) and labels with shape - (819200, 2) ## Validation Data loaded with shape: (102400, 512) and labels with shape - (102400, 2) # load existing model if exists model_dir = self.model_dir(None, params=parameters) #log_dir = os.path.join(model_dir, 'fit') log_dir = os.path.join(os.path.abspath('./output'), 'fit') parameters['accuracy'] = 0. parameters['loss'] = np.inf try: if load and os.path.exists(self.model_dir( '.h5', params=parameters)): # load old model print('loading old model...', end=' ') start_time = time.time() model = models.load_model( self.model_dir('.h5', params=parameters)) print('old model loaded in {:.2f}s.'.format(time.time() - start_time)) else: # default: build new model if self.is_train_autoencoder: optimizer = keras.optimizers.adadelta(learning_rate=1.0, rho=0.95) loss_func = keras.losses.mse encoder, decoder, autoencoder = build_autoencoder( parameters, input_shape=(self.block_size, ), gpus=self.gpus, optimizer=optimizer, loss=loss_func) model = autoencoder else: # manually defining optimizer and loss optimizer = keras.optimizers.rmsprop(lr=0.0005, rho=0.9) loss_func = keras.losses.categorical_crossentropy model = build_model(parameters, no_of_classes=no_of_classes, input_length=self.block_size, gpus=self.gpus, optimizer=optimizer, loss=loss_func) if self.is_train_autoencoder: y_train_ = x_train_ y_val_ = x_val_ print( f"Model path: \"{self.model_dir('.h5', params=parameters)}\"") monitor = 'val_loss' if self.is_train_autoencoder else 'val_acc' callbacks_list = [ callbacks.EarlyStopping(monitor=monitor, patience=50, restore_best_weights=True, min_delta=0.01), # saving model with exact name callbacks.ModelCheckpoint(self.model_dir('.h5'), monitor=monitor), callbacks.CSVLogger(filename=(self.model_dir('.log')), append=True), # saving model hparam_str in name callbacks.ModelCheckpoint(self.model_dir('.h5', params=parameters), monitor=monitor), callbacks.CSVLogger(filename=(self.model_dir( '.log', params=parameters)), append=True), ] if epochs > 2: # tensorboard logging callbacks_list.append( callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, embeddings_freq=0)) #callbacks_list.append(callbacks.TensorBoard(log_dir=log_dir)) os.makedirs(model_dir, exist_ok=True) model.summary() history = model.fit( x=x_train_, y=y_train_, epochs=epochs, batch_size=batch_size, validation_data=(x_val_, y_val_), verbose=self.verbose, callbacks=callbacks_list, ) try: split_model_path = os.path.split(self.model_name) size = len(split_model_path) keras.utils.plot_model( model, os.path.join(self.model_dir(None, params=parameters), split_model_path[size - 1] + '.png'), show_shapes=True) except Exception as e: print(f"Couln't plot diagram: {e}") if 'val_loss' in history.history: parameters['loss'] = max(history.history['val_loss']) if 'val_acc' in history.history: parameters['accuracy'] = max(history.history['val_acc']) print() self.df.loc[len(self.df)] = parameters backend.clear_session() except ValueError as ve: print('!!ERROR: {0}'.format(ve)) # raise exception if not whitelisted whitelisted_exceptions = [ 'Negative dimension size caused by subtracting', 'Error when checking target: expected ', '' ] if not list(filter(str(ve).__contains__, whitelisted_exceptions)): raise ve print("Loss: {}".format(parameters['loss'])) print("Accuracy: {:.2%}".format(parameters['accuracy'])) return parameters['loss']
def baseline(self, dataset, input_dim=784, output_dim=10, lr=0.001, epochs=100, batch_size=32, random_seed=29, mute=False, test=True, noVal=False, upload=False, dropout=True, class_curve=False): #set up random seed from numpy.random import seed seed(random_seed) from tensorflow import set_random_seed set_random_seed(random_seed) verbose = 0 if (mute): verbose = 0 else: verbose = 1 x_train, y_train = np.asarray(dataset.train_images), np.asarray( dataset.train_labels) x_val, y_val = np.asarray(dataset.validation_images), np.asarray( dataset.validation_labels) x_test, y_test = np.asarray(dataset.test_images), np.asarray( dataset.test_labels) architecture = self.architecture if (noVal): x_train = np.concatenate((x_train, x_val), axis=0) y_train = np.concatenate((y_train, y_val), axis=0) model = Sequential() model.add( Dense(architecture[0], activation='relu', input_dim=input_dim)) for i in range(1, len(architecture)): model.add(Dense(architecture[i], activation='relu')) if (dropout): model.add(Dropout(0.5)) model.add(Dense(output_dim, activation='softmax')) rmsprop = optimizers.RMSprop(lr=lr, rho=0.9, epsilon=None, decay=0.0) model.compile(loss='categorical_crossentropy', optimizer=rmsprop, metrics=['accuracy']) filepath = self.name + "baseline.best.hdf5" checkpoint = callbacks.ModelCheckpoint(filepath, monitor='val_loss', verbose=verbose, save_best_only=True, mode='auto') early_Stopping = callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=20, verbose=verbose, mode='auto') Callbacks = [checkpoint, early_Stopping] #show training curve of each class if (class_curve): class_accu = class_accuracy() Callbacks.append(class_accu) if (not noVal): model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose, callbacks=Callbacks, validation_data=(x_val, y_val), shuffle=True) else: model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose, callbacks=Callbacks, validation_data=(x_test, y_test), shuffle=True) score = 0 model.load_weights(filepath) if (test): score = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=verbose) else: score = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=verbose) if (upload): weights = [] bias = [] for i in range(0, len(model.layers)): parameters = model.layers[i].get_weights() #dropout layers doesn't have weight if (not parameters): continue weights.append(copy.deepcopy(parameters[0])) bias.append(copy.deepcopy(parameters[1])) self.weights = weights self.bias = bias return score