def __init__(self, lr, momentum): SGD.__init__(self, lr=lr, decay=1e-6, momentum=momentum, nesterov=True)
def __init__(self, n_tasks, n_features, task_type, n_layers=1, n_hidden=1000, init="glorot_uniform", batchnorm=False, dropout=0.5, activation="relu", learning_rate=.001, decay=1e-6, momentum=0.9, nesterov=False, **kwargs): super(MultiTaskDNN, self).__init__() # Store hyperparameters assert task_type in ["classification", "regression"] self.task_type = task_type self.n_features = n_features self.n_tasks = n_tasks self.n_layers = n_layers self.n_hidden = n_hidden self.init = init self.batchnorm = batchnorm self.dropout = dropout self.activation = activation self.learning_rate = learning_rate self.decay = decay self.momentum = momentum self.nesterov = nesterov self.add_input(name="input", input_shape=(self.n_features, )) prev_layer = "input" for ind, layer in enumerate(range(self.n_layers)): dense_layer_name = "dense%d" % ind activation_layer_name = "activation%d" % ind batchnorm_layer_name = "batchnorm%d" % ind dropout_layer_name = "dropout%d" % ind self.add_node(Dense(self.n_hidden, init=self.init), name=dense_layer_name, input=prev_layer) prev_layer = dense_layer_name if self.batchnorm: self.add_node(BatchNormalization(), input=prev_layer, name=batchnorm_layer_name) prev_layer = batchnorm_layer_name self.add_node(Activation(self.activation), name=activation_layer_name, input=prev_layer) prev_layer = activation_layer_name if self.dropout > 0: self.add_node(Dropout(self.dropout), name=dropout_layer_name, input=prev_layer) prev_layer = dropout_layer_name for task in range(self.n_tasks): if self.task_type == "classification": self.add_node(Dense(2, init=self.init, activation="softmax"), name="dense_head%d" % task, input=prev_layer) elif self.task_type == "regression": self.add_node(Dense(1, init=self.init), name="dense_head%d" % task, input=prev_layer) self.add_output(name="task%d" % task, input="dense_head%d" % task) loss_dict = {} for task in range(self.n_tasks): taskname = "task%d" % task if self.task_type == "classification": loss_dict[taskname] = "binary_crossentropy" elif self.task_type == "regression": loss_dict[taskname] = "mean_squared_error" sgd = SGD(lr=self.learning_rate, decay=self.decay, momentum=self.momentum, nesterov=self.nesterov) self.compile(optimizer=sgd, loss=loss_dict)
''' Import keras to build a DL model ''' from keras.models import Sequential from keras.layers.core import Dense, Dropout, Activation print 'Building model using SGD(lr=0.1)' ''' 1. Model using large learning rate ''' model_large = Sequential() model_large.add(Dense(128, input_dim=200)) model_large.add(Activation('sigmoid')) model_large.add(Dense(256)) model_large.add(Activation('sigmoid')) model_large.add(Dense(5)) model_large.add(Activation('softmax')) ''' set the learning rate of SGD optimizer to 0.1 ''' from keras.optimizers import SGD, Adam, RMSprop, Adagrad sgd010 = SGD(lr=0.1, momentum=0.0, decay=0.0, nesterov=False) model_large.compile(loss='categorical_crossentropy', optimizer=sgd010, metrics=['accuracy']) history_large = model_large.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=True, validation_split=0.1) loss_large = history_large.history.get('loss') acc_large = history_large.history.get('acc')
def main(): init_detector() args = parser.parse_args() train_path = args.train_dir test_path = args.test_dir if args.gpu: config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.per_process_gpu_memory_fraction = 0.85 K.tensorflow_backend.set_session(tf.Session(config=config)) img, age, gender = load_data(train_path) x_train_data = img y_train_age = np_utils.to_categorical(age, AGE_CATEGORY) y_train_gender = np_utils.to_categorical(gender, 2) img, age, gender = load_data(test_path) x_test_data = img y_test_age = np_utils.to_categorical(age, AGE_CATEGORY) y_test_gender = np_utils.to_categorical(gender, 2) inputs = Input(shape=INPUT_SHAPE) x = Conv2D(32, (3, 3), activation='relu')(inputs) x = MaxPooling2D(2, 2)(x) x = Conv2D(32, (3, 3), activation='relu')(x) x = MaxPooling2D(2, 2)(x) x = Conv2D(64, (3, 3), activation='relu')(x) x = MaxPooling2D(2, 2)(x) x = Conv2D(64, (3, 3), activation='relu')(x) x = BatchNormalization(axis=-1)(x) pool = AveragePooling2D(pool_size=(4, 4), strides=(1, 1), padding="same")(x) flatten = Flatten()(pool) predictions_g = Dense(units=2, kernel_initializer=WEIGHT_INIT, use_bias=USE_BIAS, kernel_regularizer=l2(WEIGHT_DECAY), activation="softmax")(flatten) predictions_a = Dense(units=AGE_CATEGORY, kernel_initializer=WEIGHT_INIT, use_bias=USE_BIAS, kernel_regularizer=l2(WEIGHT_DECAY), activation="softmax")(flatten) model = Model(inputs=inputs, outputs=[predictions_g, predictions_a]) model.summary() sgd = SGD(lr=0.1, momentum=0.9, nesterov=True) model.compile( optimizer=sgd, loss=["categorical_crossentropy", "categorical_crossentropy"], metrics=['accuracy']) callbacks = [ EarlyStopping(monitor='val_loss', patience=100, verbose=0, mode='auto', baseline=None, restore_best_weights=True), ModelCheckpoint("checkpoints/weights.{epoch:02d}-{val_loss:.2f}.hdf5", monitor="val_loss", verbose=1, save_best_only=True, mode="auto") ] hist = model.fit(x_train_data, [y_train_gender, y_train_age], batch_size=BATCH_SIZE, epochs=EPOCHS, callbacks=callbacks, validation_data=(x_test_data, [y_test_gender, y_test_age])) print('Train complete.') with K.get_session() as sess: # frozen graph additional_nodes = ['input_1', 'dense_1/Softmax', 'dense_2/Softmax'] frozen_graph = freeze_session(sess, output_names=additional_nodes) # save model to pb file tf.train.write_graph(frozen_graph, "./", "age_gender_v1.pb", as_text=False) tf.train.write_graph(frozen_graph, "./", "age_gender_v1.pbtxt", as_text=True)
kernel_initializer='VarianceScaling')) model.add(Dropout(dropout)) model.add( Dense(hidden4, activation=activation, name='dense_4', kernel_initializer='VarianceScaling')) model.add( Dense(len(emotions), activation='softmax', trainable=trainable, name='dense_55')) for learn_rate in learn_rates: sgd = SGD(lr=learn_rate, decay=1e-6, momentum=0.9, nesterov=False) rmsdrop = keras.optimizers.RMSprop(lr=learn_rate, rho=0.9, epsilon=None, decay=0.0) adagrad = keras.optimizers.Adagrad(lr=learn_rate, epsilon=None, decay=0.0) adadelta = keras.optimizers.Adadelta(lr=learn_rate, rho=0.95, epsilon=None, decay=0.0) adam = keras.optimizers.Adam(lr=learn_rate, beta_1=0.9, beta_2=0.999,
model.add( Convolution2D(32, 7, 7, border_mode='same', input_shape=X_train[0].shape)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.2)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(classes)) model.add(Activation('softmax')) '''setting optimizer''' learning_rate = 0.01 learning_decay = 0.01 / 32 sgd = SGD(lr=learning_rate, decay=learning_decay, momentum=0.9, nesterov=True) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # check parameters of every layers model.summary() ''' training''' batch_size = 128 epoch = 32 # validation data comes from training data # model.fit(X_train, Y_train, batch_size=batch_size, # nb_epoch=epoch, validation_split=0.1, shuffle=True) # validation data comes from testing data fit_log = model.fit(X_train,
i = cake(128, 256, 2, 2)(i) # 4*4 # conv5_x i = cake(256, 512, 2, 2)(i) # 2*2 i = BatchNormalization(axis=-1)(i) i = relu(i) i = AveragePooling2D(pool_size=(2, 2), padding='same')(i) i = Flatten()(i) i = Dense(12)(i) i = Activation('softmax')(i) model = Model(outputs=i, inputs=inp) sgd = SGD(lr=0.1, decay=0.04, momentum=0.9, nesterov=False) model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy']) # # # # # # model specification scripts------------------------------------------------------------ # import the image data from the directory which stores each image under its category folder # if you want more details about what the data should be arranged for imagedatagenerator to successfully import into the system, see "https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html" train_datagen = ImageDataGenerator(featurewise_center=True) test_datagen = ImageDataGenerator(rescale=1. / 255) train_generator = train_datagen.flow_from_directory(train_data_dir, target_size=(img_width, img_height), batch_size=batch_size,
def main(): config = configparser.ConfigParser() config_path = os.path.join("configurations", "master_configuration.ini") config.read(config_path) print("\nReading info from configuration:") args = argument_parser() if args["operation"] == "Training": print("\nRunnning in training setting!\n") NUM_TRAINING = int(config["TRAINING"]["NUM_TRAINING"]) NUM_VALIDATION = int(config["TRAINING"]["NUM_VALIDATION"]) NUM_EPOCHS = int(config["TRAINING"]["NUM_EPOCHS"]) elif args["operation"] == "Development": print("\nRunnning in development setting!\n") NUM_TRAINING = int(config["DEVELOPMENT"]["NUM_TRAINING"]) NUM_VALIDATION = int(config["DEVELOPMENT"]["NUM_VALIDATION"]) NUM_EPOCHS = int(config["DEVELOPMENT"]["NUM_EPOCHS"]) else: print("\nError: Operation should be either 'Training' or 'Development'") print("Exiting!\n") sys.exit(1) if args["epoch"] != "Default": try: NUM_EPOCHS = int(args["epoch"]) except ValueError: print("\nError: Epoch should be an integer.") print("Exiting!\n") sys.exit(1) BATCH_SIZE = int(config["DEFAULT"]["BATCH_SIZE"]) IMAGE_WIDTH = int(config["DEFAULT"]["IMAGE_WIDTH"]) IMAGE_HEIGHT = int(config["DEFAULT"]["IMAGE_HEIGHT"]) IMAGE_DEPTH = int(config["DEFAULT"]["IMAGE_DEPTH"]) CLASS_NAMES = config["DEFAULT"]["CLASS_NAMES"].split() FEATURE_FILE_TRAINING = config["DEFAULT"]["FEATURE_FILE_TRAINING"] LABEL_FILE_TRAINING = config["DEFAULT"]["LABEL_FILE_TRAINING"] FEATURE_FILE_VALIDATION = config["DEFAULT"]["FEATURE_FILE_VALIDATION"] LABEL_FILE_VALIDATION = config["DEFAULT"]["LABEL_FILE_VALIDATION"] WEIGHTS = np.array(list(map(float, config["DEFAULT"]["WEIGHTS"].split()))) print("NUM_TRAINING: {}".format(NUM_TRAINING)) print("NUM_VALIDATION: {}".format(NUM_VALIDATION)) print("NUM_EPOCHS: {}".format(NUM_EPOCHS)) print("BATCH_SIZE: {}".format(BATCH_SIZE)) print("IMAGE_WIDTH: {}".format(IMAGE_WIDTH)) print("IMAGE_HEIGHT: {}".format(IMAGE_HEIGHT)) print("IMAGE_DEPTH: {}".format(IMAGE_DEPTH)) print("CLASS_NAMES: {}".format(CLASS_NAMES)) print("FEATURE_FILE_TRAINING: {}".format(FEATURE_FILE_TRAINING)) print("LABEL_FILE_TRAINING: {}".format(LABEL_FILE_TRAINING)) print("FEATURE_FILE_VALIDATION: {}".format(FEATURE_FILE_VALIDATION)) print("LABEL_FILE_VALIDATION: {}".format(LABEL_FILE_VALIDATION)) print("WEIGHTS: {}".format(WEIGHTS)) print() datasequence_training = DataSequence(feature_file=FEATURE_FILE_TRAINING, label_file=LABEL_FILE_TRAINING, image_width=IMAGE_WIDTH, image_height=IMAGE_HEIGHT, image_depth=IMAGE_DEPTH, num_classes=len(CLASS_NAMES), max_index=NUM_TRAINING, batch_size=BATCH_SIZE) datasequence_validation = DataSequence(feature_file=FEATURE_FILE_VALIDATION, label_file=LABEL_FILE_VALIDATION, image_width=IMAGE_WIDTH, image_height=IMAGE_HEIGHT, image_depth=IMAGE_DEPTH, num_classes=len(CLASS_NAMES), max_index=NUM_VALIDATION, batch_size=BATCH_SIZE) # Note: num_filters needs to be 16 or less for batch size of 5 (for 6 GB memory) # Compile the model input_tensor = Input((IMAGE_WIDTH, IMAGE_HEIGHT, IMAGE_DEPTH)) model = get_unet_model(input_tensor=input_tensor, num_classes=len(CLASS_NAMES), num_filters=64, dropout=0.25, batchnorm=True) model.compile(optimizer=SGD(lr=1e-5, decay=0.0), loss=focal_loss(), metrics=['accuracy']) model_and_weights = os.path.join("saved_models", "model_and_weights.hdf5") # If weights exist, load them before continuing training continue_training = True if(os.path.isfile(model_and_weights) and continue_training): print("Old weights found!") try: model.load_weights(model_and_weights) print("Old weights loaded successfully!") except: print("Old weights couldn't be loaded successfully, will continue!") # Traing the model history = train_model(model=model, X=datasequence_training, y=datasequence_validation, num_training=NUM_TRAINING, num_validation=NUM_VALIDATION, model_path=model_and_weights, num_epochs=NUM_EPOCHS, batch_size=BATCH_SIZE) # Plot the history loss_path = os.path.join("plots", "loss_vs_epoch.pdf") plot_history(history, quantity='loss', plot_title='Loss', y_label='Loss', plot_name=loss_path) accuracy_path = os.path.join("plots", "accuracy_vs_epoch.pdf") plot_history(history, quantity='acc', plot_title='Accuracy', y_label='Accuracy', plot_name=accuracy_path)
# convert the labels from integers to vectors trainY = LabelBinarizer().fit_transform(trainY) testY = LabelBinarizer().fit_transform(testY) # image data generator for data augmentation augmentation = ImageDataGenerator(rotation_range=30, width_shift_range=0.1, height_shift_range=0.1, shear_range=0.2, zoom_range=0.2, horizontal_flip=True, fill_mode="nearest") # initializing model print("[INFO] compiling model...") optimizer = SGD(lr=0.05) model = MiniVGGNet.build(width=64, height=64, depth=3, classes=len(classNames)) model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"]) # train the network print("[INFO] training network...") H = model.fit_generator(augmentation.flow(trainX, trainY, batch_size=32), validation_data=(testX, testY), steps_per_epoch=len(trainX) // 32, epochs=100, verbose=1) # evaluate network print("[INFO] evaluating network...")
Y_train = np_utils.to_categorical(y_train, nb_classes) Y_test = np_utils.to_categorical(y_test, nb_classes) model = Sequential() model.add(Dense(512, input_shape=(nb_data,), init='normal', name='dense1')) # a sample is a row 28*28 model.add(Activation('relu', name='relu1')) model.add(Dropout(0.2, name='dropout1')) model.add(Dense(512, init='normal', name='dense2')) model.add(Activation('relu', name='relu2')) model.add(Dropout(0.2, name='dropout2')) model.add(Dense(10, init='normal', name='dense3')) model.add(Activation('softmax', name='softmax1')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.001), metrics=['accuracy']) tb_cb = keras.callbacks.TensorBoard(log_dir=log_filepath, write_images=1, histogram_freq=1) # 设置log的存储位置,将网络权值以图片格式保持在tensorboard中显示,设置每一个周期计算一次网络的 #权值,每层输出值的分布直方图 history = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, callbacks=[tb_cb], validation_data=(X_test, Y_test)) score = model.evaluate(X_test, Y_test, verbose=0) print('Test score:', score[0]) print('Test accuracy;', score[1])
test_x = test_x.astype("float") / 255.0 # Convert the labels from integers to vectors lb = LabelBinarizer() train_y = lb.fit_transform(train_y) test_y = lb.transform(test_y) # Initialize the label names for the CIFAR-10 dataset label_names = [ "airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck" ] # Initialize the SGD optimizer print("[INFO]: Compiling model....") optimizer = SGD(lr=0.01, decay=0.01 / 40, momentum=0.9, nesterov=True) model = MiniVGGNet.build(width=32, height=32, depth=3, classes=10) model.compile(loss="categorical_crossentropy", optimizer=optimizer, metrics=["accuracy"]) # Construct the set of callbacks fig_path = os.path.sep.join([args["output"], "{}.png".format(os.getpid())]) json_path = os.path.sep.join([args["output"], "{}.json".format(os.getpid())]) callbacks = [TrainingMonitor(fig_path, json_path)] # Construct the callback to save only the 'best' model to disk based on the validation loss checkpoint = ModelCheckpoint(args['weights'], monitor="val_loss", mode="min", save_best_only=True,
model = Sequential() model.add(Dense(40, input_dim=2 * 1 * nb_classes, init='he_normal')) model.add(Activation('relu')) model.add(Dense(40, init='he_normal')) model.add(Activation('relu')) model.add(Dense(40, init='he_normal')) model.add(Activation('relu')) model.add(Dense(3, init='he_normal')) model.add(Activation('softmax')) rms = RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=1.5) adagrad = Adagrad(clipnorm=1.5) adam = Adam() adadelta = Adadelta() sgd = SGD(lr=0.01, decay=1e-6, momentum=0.95, nesterov=True, clipnorm=1.0) model.compile(loss='poisson', optimizer=adam) model.fit(C, Y, nb_epoch=10, batch_size=100, show_accuracy=True, verbose=0, validation_data=(C, Y)) # ----------------------------------------------------- pred_class = model.predict_classes(C, batch_size=200, verbose=0) s_nn_hat = hstack((zeros(1), pred_class, zeros(1))) x_nn_hat = N_DUDE.denoise_with_s(z[i], s_nn_hat, 1)
resize_p = ResizePreprocessor(32, 32) image2array_p = ImageToArrayPreprocessor() loader = DatasetLoader(preprocessors=[resize_p, image2array_p]) (data, labels) = loader.load(imageFiles, verbose=500) data = data.astype("float") / 255.0 (trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.25, random_state=17) trainY = LabelBinarizer().fit_transform(trainY) testY = LabelBinarizer().fit_transform(testY) print("[INFO] compiling model...") opt = SGD(lr=0.005) model = ShallowNet.build(width=32, height=32, depth=3, classes=3) model.compile(loss="categorical_crossentropy", optimizer=opt, metrics=["accuracy"]) print("[INFO] training network...") N_epoch = 40 H = model.fit(trainX, trainY, validation_data=(testX, testY), batch_size=32, epochs=N_epoch, verbose=1) print("[INFO] predict test data...")
# load dataset from datasets import load_data x, y = load_data(args.dataset) n_clusters = len(np.unique(y)) init = 'glorot_uniform' #初始化器的选择 pretrain_optimizer = 'adam' # setting parameters if args.dataset == 'mnist' or args.dataset == 'fmnist': update_interval = 140 pretrain_epochs = 300 init = VarianceScaling( scale=1. / 3., mode='fan_in', distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in) pretrain_optimizer = SGD(lr=1, momentum=0.9) elif args.dataset == 'reuters10k': update_interval = 30 pretrain_epochs = 50 init = VarianceScaling( scale=1. / 3., mode='fan_in', distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in) pretrain_optimizer = SGD(lr=1, momentum=0.9) elif args.dataset == 'jd': update_interval = 30 pretrain_epochs = 50 init = VarianceScaling( scale=1. / 3., mode='fan_in', distribution='uniform') # [-limit, limit], limit=sqrt(1./fan_in) pretrain_optimizer = SGD(lr=1, momentum=0.9) elif args.dataset == 'jdReviewWithLabel':
from keras.utils import to_categorical batch_size = 128 epochs = 20 learning_rate = 0.5 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.reshape([x_train.shape[0], -1]).astype('float32') / 255 y_train = to_categorical(y_train, num_classes=10) x_test = x_test.reshape([x_test.shape[0], -1]).astype('float32') / 255 y_test = to_categorical(y_test, num_classes=10) model = Sequential() model.add( Dense(10, activation='sigmoid', input_shape=(784, ), kernel_initializer=RandomUniform(minval=-0.01, maxval=0.01))) model.summary() model.compile(loss='mean_squared_error', optimizer=SGD(lr=learning_rate), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test))
num_users, num_items = train.shape print("Load data done [%.1f s]. #user=%d, #item=%d, #train=%d, #test=%d" % (time() - t1, num_users, num_items, train.nnz, len(testRatings))) # Build model model = get_model(num_users, num_items, mf_dim, layers, reg_layers, reg_mf) if learner.lower() == "adagrad": model.compile(optimizer=Adagrad(lr=learning_rate), loss='binary_crossentropy') elif learner.lower() == "rmsprop": model.compile(optimizer=RMSprop(lr=learning_rate), loss='binary_crossentropy') elif learner.lower() == "adam": model.compile(optimizer=Adam(lr=learning_rate), loss='binary_crossentropy') else: model.compile(optimizer=SGD(lr=learning_rate), loss='binary_crossentropy') # Init performance (hits, ndcgs) = evaluate_model(model, testRatings, testNegatives, topK, evaluation_threads) hr, ndcg = np.array(hits).mean(), np.array(ndcgs).mean() print('Init: HR = %.4f, NDCG = %.4f' % (hr, ndcg)) best_hr, best_ndcg, best_iter = hr, ndcg, -1 if out_model > 0: model.save_weights(model_out_file, overwrite=True) # Training model num_epochs = 10 for epoch in range(num_epochs): t1 = time() # Generate training instances
def run(args): lr = args.lr epochs = args.epochs decay = args.decay momentum = args.momentum h5file = args.model test_set_path = args.test hist = args.hist dataset = pd.read_csv( os.path.join('/home', 'wvillegas', 'dataset-mask', 'full_masks.csv')) from utils_fcn import DataGeneratorMobileNet from sklearn.model_selection import train_test_split X_train, X_test, Y_train, Y_test = train_test_split(dataset['orig'], dataset['mask'], test_size=0.2, random_state=1) partition = {'train': list(X_train), 'test': list(X_test)} img_list = list(X_train) + list(X_test) mask_list = list(Y_train) + list(Y_test) labels = dict(zip(img_list, mask_list)) img_path = os.path.join('/home', 'wvillegas', 'dataset-mask', 'dataset_resize', 'images_resize') masks_path = os.path.join('/home', 'wvillegas', 'dataset-mask', 'dataset_resize', 'masks_resize') batch_size = 4 train_generator = DataGeneratorMobileNet(batch_size=batch_size, img_path=img_path, labels=labels, list_IDs=partition['train'], n_channels=3, n_channels_label=1, shuffle=True, mask_path=masks_path) from keras.applications import MobileNet from keras.layers import Conv2DTranspose, Conv2D, Add from keras import Model net = MobileNet(include_top=False, weights=None) net.load_weights( '/home/wvillegas/DLProjects/BudClassifier/cmdscripts/modelosV2/mobilenet_weights_detection.h5', by_name=True) for layer in net.layers: layer.trainable = True predict = Conv2D(filters=1, kernel_size=1, strides=1)(net.output) deconv2 = Conv2DTranspose(filters=1, kernel_size=4, strides=2, padding='same', use_bias=False)(predict) pred_conv_dw_11_relu = Conv2D(filters=1, kernel_size=1, strides=1)( net.get_layer('conv_dw_11_relu').output) fuse1 = Add()([deconv2, pred_conv_dw_11_relu]) pred_conv_pw_5_relu = Conv2D(filters=1, kernel_size=1, strides=1)( net.get_layer('conv_pw_5_relu').output) deconv2fuse1 = Conv2DTranspose(filters=1, kernel_size=4, strides=2, padding='same', use_bias=False)(fuse1) fuse2 = Add()([deconv2fuse1, pred_conv_pw_5_relu]) deconv8 = Conv2DTranspose(filters=1, kernel_size=16, strides=8, padding='same', use_bias=False)(fuse2) fcn = Model(inputs=net.input, outputs=deconv8) from keras.optimizers import SGD sgd = SGD(lr=lr, momentum=momentum, decay=decay) fcn.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) history = fcn.fit_generator(generator=train_generator, use_multiprocessing=True, workers=6, epochs=epochs) fcn.save(os.path.join(h5file)) test_csv = pd.DataFrame({'x': X_test, 'y': Y_test}) test_csv.to_csv(test_set_path, header=None) test_csv = pd.DataFrame(history.history) test_csv.to_csv(hist)
'data_dir': ds_path + '/', 'fullds_data_dir': ds_path + '/', #'data_dir': ds_path + '/malware_selected_1gram_mini.pkl', #'fullds_data_dir':ds_path + '/malware_selected_1gram.pkl', 'data_target_list': [1, 2, 3, 4, 5, 6, 7, 8, 9], #'epochs': 50, #'epochs': 200, 'epochs': 1000, 'batch': 32, 'store_history': True, 'shuffle_batches': True, 'autoencoder_configs': { 'hidden_layer_activation': 'relu', 'output_layer_activation': 'relu', 'loss_function': 'mse', 'optimizer': SGD(lr=0.01), 'discard_decoder_function': True }, 'mlp_configs': { 'activation': 'sigmoid', #'activation' : 'softmax', 'loss_function': 'categorical_crossentropy', 'optimizer': SGD(lr=0.01), 'use_last_dim_as_classifier': False, 'classifier_dim': 9 } } MAP_DIMS = { 'AE_BIGRAMA_7L_MINIDS_UNDER_01': [9216, 5530, 4978, 4426, 3874, 3322, 2770, 2217],
def testmodels_test(X_tr, y_tr, X_te, archs, actfn='relu', last_act='softmax', reg_coeffs=[0.0], num_epoch=30, batch_size=1000, sgd_lr=1e-5, sgd_decays=[0.0], sgd_moms=[0.0], sgd_Nesterov=False, EStop=False, verbose=0): [n_tr, d] = X_tr.shape [n_te, d] = X_te.shape call_ES = EarlyStopping(monitor='val_acc', patience=6, verbose=1, mode='auto') for arch in archs: for reg_coeff in reg_coeffs: for sgd_decay in sgd_decays: for sgd_mom in sgd_moms: # Generate Model model = genmodel(num_units=arch, actfn=actfn, reg_coeff=reg_coeff, last_act=last_act) # Compile Model sgd = SGD(lr=sgd_lr, decay=sgd_decay, momentum=sgd_mom, nesterov=sgd_Nesterov) model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) # Train Model if EStop: model.fit(X_tr, y_tr, nb_epoch=num_epoch, batch_size=batch_size, verbose=verbose, callbacks=[call_ES], validation_split=0.1, validation_data=None, shuffle=True) else: model.fit(X_tr, y_tr, nb_epoch=num_epoch, batch_size=batch_size, verbose=verbose) probability = model.predict_proba(X_te, batch_size=batch_size, verbose=verbose) return probability
def vgg16_model(img_rows, img_cols, channel=1, num_classes=None): """VGG 16 Model for Keras Model Schema is based on https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3 ImageNet Pretrained Weights https://drive.google.com/file/d/0Bz7KyqmuGsilT0J5dmRCM0ROVHc/view?usp=sharing Parameters: img_rows, img_cols - resolution of inputs channel - 1 for grayscale, 3 for color num_classes - number of categories for our classification task """ model = Sequential() model.add(ZeroPadding2D((1, 1), input_shape=(channel, img_rows, img_cols))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(64, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(128, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(256, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(ZeroPadding2D((1, 1))) model.add(Convolution2D(512, 3, 3, activation='relu')) model.add(MaxPooling2D((2, 2), strides=(2, 2))) # Add Fully Connected Layer model.add(Flatten()) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(4096, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(1000, activation='softmax')) # Loads ImageNet pre-trained data model.load_weights('D:\Projeto\Datase\vgg16_weights.h5') # Truncate and replace softmax layer for transfer learning model.layers.pop() model.outputs = [model.layers[-1].output] model.layers[-1].outbound_nodes = [] model.add(Dense(num_classes, activation='softmax')) # Uncomment below to set the first 10 layers to non-trainable (weights will not be updated) #for layer in model.layers[:10]: # layer.trainable = False # Learning rate is changed to 0.001 sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy']) return model
def train( data, dims, resolution=0.2, alpha=1.0, tol=0.005, init='glorot_uniform', n_clusters=None, n_neighbors=15, pretrain_epochs=300, batch_size=256, activation='relu', actincenter='tanh', drop_rate_sae=0.2, is_stacked=True, use_early_stop=True, use_ae_weights=False, save_encoder_weights=False, save_dir='tmp_result', max_iter=1000, epochs_fit=4, num_cores=4, use_gpu=False, random_seed=1, verbose=True, ): if isinstance(data, AnnData): adata = data else: adata = AnnData(data) assert dims[0] == adata.shape[-1], \ 'The number of columns of data needs to be equal to the first element of dims!' random.seed(random_seed) np.random.seed(random_seed) tf.set_random_seed(random_seed) total_cpu = multiprocessing.cpu_count() num_cores = int(num_cores) if total_cpu > int(num_cores) else int( ceil(total_cpu / 2)) print('The number of threads in your computer is', total_cpu) print('The number of threads used is', num_cores) if use_gpu: os.environ["CUDA_VISIBLE_DEVICES"] = "0" else: os.environ["CUDA_VISIBLE_DEVICES"] = "-1" kb.set_session( tf.Session( config=tf.ConfigProto(intra_op_parallelism_threads=num_cores, inter_op_parallelism_threads=num_cores))) if not use_ae_weights and os.path.isfile( os.path.join(save_dir, "ae_weights.h5")): os.remove(os.path.join(save_dir, "ae_weights.h5")) tic = get_time() # record time ######################## desc = DescModel(dims=dims, x=adata.X, alpha=alpha, tol=tol, init=init, n_clusters=n_clusters, louvain_resolution=resolution, n_neighbors=n_neighbors, pretrain_epochs=pretrain_epochs, batch_size=batch_size, random_seed=random_seed, activation=activation, actincenter=actincenter, drop_rate_SAE=drop_rate_sae, is_stacked=is_stacked, use_earlyStop=use_early_stop, use_ae_weights=use_ae_weights, save_encoder_weights=save_encoder_weights, save_dir=save_dir) desc.compile(optimizer=SGD(0.01, 0.9), loss='kld') embedded, prob = desc.fit(maxiter=max_iter, epochs_fit=epochs_fit) print("The desc has been trained successfully!!!!!!") if verbose: print("The summary of desc model is:") desc.model.summary() print("The runtime is ", get_time() - tic) adata.obsm['embedded'] = embedded adata.obsm['prob'] = prob adata.obs['ident'] = np.argmax(prob, axis=1) adata.obs['max_prob'] = _get_max_prob(prob) return adata
def main(args): # set the image parameters img_rows = args.img_rows img_cols = args.img_cols img_dim_color = args.img_channels # mix_prop = 1.0 # set the value of the mixing proportion ############################################################################################################# ################################## DEFINING MODEL ########################################################## ############################################################################################################## model_alex = cnn_hybrid_color_single(img_rows, img_cols, img_dim_color) # load the model # model_final = Model(model_alex.input, model_alex.output) # specify the input and output of the model model_final = model_alex print(model_final.summary()) # print the model summary plot_model(model_final, to_file='./NIN_hybrid_bin_resnet_1x1-class', show_shapes=True) # save the model summary as a png file lr = args.learning_rate # set the learning rate # set the optimizer optimizer = SGD(lr=lr, decay=1e-6, momentum=0.9) # model compilation model_final.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) # print the value of the learning rate print(K.get_value(optimizer.lr)) # -------------------------------------------------- ############################################################################################################# ########################## GETTING TRAINING DATA AND TESTING DATA ########################################## ############################################################################################################## # get the training data by calling the pairs function # read the training data train_pairs_r, training_data_r, training_label_r = read_pairs( args.tr_img_lab_r) train_pairs_l, training_data_l, training_label_l = read_pairs( args.tr_img_lab_l) assert len(training_data_r) == len(training_data_l) # combine the left and right image in the training data to make a X x Y x 6 tensor training_data = [] for i in range(len(training_data_r)): # define the stereo pair stereo_pair = [training_data_r[i], training_data_l[i]] training_data.append(stereo_pair) batch_num = 0 # initialize the live samples and fake samples live_samples_ub = 0 attack_samples_ub = 0 live_samples = [] live_labels = [] attack_samples = [] attack_labels = [] # separate the live samples and fake samples to balance the both classes, i.e. live class and fake class assert len(training_label_r) == len(training_label_l) for i in range(len(training_data)): if training_label_r[i] == 0: live_samples.append(training_data[i]) live_labels.append(training_label_r[i]) live_samples_ub += 1 elif (training_label_r[i] == 1) | (training_label_r[i] == 3) | ( training_label_r[i] == 4): # cut photo attack removed attack_samples.append(training_data[i]) attack_labels.append(training_label_r[i]) attack_samples_ub += 1 print("Live samples are %g ,\t attack samples are %g" % (live_samples_ub, attack_samples_ub)) # compute the difference; the live samples are always less than the fake samples in our case diff = 0 if live_samples_ub < attack_samples_ub: # compute the ratio diff = np.int(attack_samples_ub / live_samples_ub) print("The difference is :%g " % (diff)) else: ValueError("The fake samples are less than then live samples") # number of times the dataset has to be copied: live_samples_b = live_samples live_labels_b = live_labels for i in range(diff - 1): # print("length before balancing: %g" %len(live_samples_b)) sl_copy = live_samples.copy() ll_copy = live_labels.copy() live_samples_b = live_samples_b + sl_copy live_labels_b = live_labels_b + ll_copy # print("length after balancing: %g" % len(live_samples_b)) # balanced data training_data_balanced = live_samples_b + attack_samples training_label_balanced = live_labels_b + attack_labels print("Balanced data samples: %g" % len(training_data_balanced)) # get the length of the training data len_tr = len(training_data_balanced) # get the number equal to the length of the training data indices_tr = np.arange(len_tr) np.random.shuffle(indices_tr) # initialize the image counter images_read = 0 train_img_data_r = [] train_img_data_l = [] for i in indices_tr: if training_label_balanced[i] > 0: training_label_balanced[i] = 1 train_img_data_r.append( [training_data_balanced[i][0], training_label_balanced[i]]) # read the right image train_img_data_l.append( [training_data_balanced[i][1], training_label_balanced[i]]) # read the left image # print(training_data_balanced[i][1]) # cv2.imshow('img1', cv2.imread(training_data_balanced[i][0])) # cv2.waitKey() # cv2.imshow('img2', cv2.imread(training_data_balanced[i][1])) # cv2.waitKey() images_read += 1 sys.stdout.write('train images read = {0}\r'.format(images_read)) sys.stdout.flush() ############################################################################################################ # read the test data test_pairs, test_data_r, test_labels_r = read_pairs(args.tst_img_lab_r) test_pairs, test_data_l, test_labels_l = read_pairs(args.tst_img_lab_l) assert len(test_data_r) == len(test_data_l) # combine the left and right image in the training data to make a X x Y x 6 tensor test_data = [] for i in range(len(test_data_r)): # define the stereo pair stereo_pair_t = [test_data_r[i], test_data_l[i]] test_data.append(stereo_pair_t) test_labels = test_labels_r images_read = 0 # get the length of the training data len_test = len(test_data) # get the number equal to the length of the training data indices_test = np.arange(len_test) test_img_data_r = [] test_img_data_l = [] for i in indices_test: if test_labels[i] > 0: test_labels[i] = 1 test_img_data_r.append([test_data[i][0], test_labels[i]]) # read the right test image test_img_data_l.append([test_data[i][1], test_labels[i]]) # red the left test image images_read += 1 sys.stdout.write('test images read = {0}\r'.format(images_read)) sys.stdout.flush() ##################################################################################################### # make all the data in panda data frame format train_df_r = pd.DataFrame(train_img_data_r) train_df_r.columns = ['id_r', 'label'] train_df_l = pd.DataFrame(train_img_data_l) train_df_l.columns = ['id_l', 'label'] test_df_r = pd.DataFrame(test_img_data_r) test_df_r.columns = ['id_r', 'label'] test_df_l = pd.DataFrame(test_img_data_l) test_df_l.columns = ['id_l', 'label'] ########################################################################################################333 datagen = image.ImageDataGenerator() train_generator_r = datagen.flow_from_dataframe( dataframe=train_df_r, directory=None, x_col='id_r', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=True, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest', drop_duplicates=False) train_generator_l = datagen.flow_from_dataframe( dataframe=train_df_l, directory=None, x_col='id_l', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=True, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest', drop_duplicates=False) test_datagen = image.ImageDataGenerator() test_generator_r = test_datagen.flow_from_dataframe( dataframe=test_df_r, directory=None, x_col='id_r', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=False, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest') test_generator_l = test_datagen.flow_from_dataframe( dataframe=test_df_l, directory=None, x_col='id_l', y_col='label', has_ext=True, batch_size=args.batch_size, seed=42, shuffle=False, class_mode="sparse", target_size=(args.img_rows, args.img_cols), color_mode='grayscale', interpolation='nearest') ############################################################################################################# batch_num = 0 while batch_num < args.max_epochs: start_time = time.time() # initialize the clock acc = [] loss = [] sub_count = 0 total_batch = train_generator_r.n // train_generator_r.batch_size for i in range(train_generator_r.n // train_generator_r.batch_size): x1, y1 = next(train_generator_r) x2, y2 = next(train_generator_l) # only for DP-3D for comparison # disparity_final = [] # # for j in range(x1.shape[0]): # img1 = np.asarray(x1[j]) # # img1 = cv2.resize(img1, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # img2 = np.asarray(x2[j]) # # img2 = cv2.resize(img2, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # # disparity = cv2.subtract(img1, img2) # # der_k = np.asarray([[1.0, 2.0, 1.0], # [0.0, 0.0, 0.0], # [-1.0, -2.0, -1.0]]) # # der = cv2.filter2D(img1, -1, kernel=der_k) # # disparity_f = disparity / (der + 0.005) # # disparity_final.append(disparity_f) # # # disparity_final = np.asarray(disparity_final).astype('float32') # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1) x1 = x1.astype('float32') / 255 x2 = x2.astype('float32') / 255 y = y1 tr_acc1 = model_final.fit([x1, x2], y, epochs=1, verbose=0) acc.append(tr_acc1.history['acc'][0]) loss.append(tr_acc1.history['loss'][0]) sub_count += 1 sys.stdout.write('batch_count = {0} of {1} \r'.format( sub_count, total_batch)) sys.stdout.flush() train_acc = np.sum(np.asarray(acc)) * 100 / ( train_generator_r.n // train_generator_r.batch_size) train_loss = np.sum(np.asarray(loss)) * 100 / ( train_generator_r.n // train_generator_r.batch_size) print('training_acc: {0} \t training_loss: {1}'.format( train_acc, train_loss)) print( '______________________________________________________________________' ) print('Running the evaluations') test_acc = [] test_loss = [] sub_count = 0 for i in range(test_generator_r.n // test_generator_r.batch_size): x1, y1 = next(test_generator_r) x2, y2 = next(test_generator_l) # only for DP-3D for comparison # disparity_final = [] # # for j in range(x1.shape[0]): # img1 = np.asarray(x1[j]) # # img1 = cv2.resize(img1, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # img2 = np.asarray(x2[j]) # # img2 = cv2.resize(img2, (img_rows, img_cols), # # interpolation=cv2.INTER_AREA) # # # disparity = cv2.subtract(img1, img2) # # der_k = np.asarray([[1.0, 2.0, 1.0], # [0.0, 0.0, 0.0], # [-1.0, -2.0, -1.0]]) # # der = cv2.filter2D(img1, -1, kernel=der_k) # # disparity_f = disparity / (der + 0.005) # # disparity_final.append(disparity_f) # # # disparity_final = np.asarray(disparity_final).astype('float32') # disparity_final = np.expand_dims(np.asarray(disparity_final).astype('float32'), axis=-1) x1 = x1.astype('float32') / 255 x2 = x2.astype('float32') / 255 y = y1 tst_loss, tst_acc1 = model_final.evaluate([x1, x2], y, verbose=0) test_acc.append(tst_acc1) test_loss.append(tst_loss) sub_count += 1 sys.stdout.write('epoch_count = {0}\r'.format(sub_count)) sys.stdout.flush() test_acc = np.sum(np.asarray(test_acc)) * 100 / ( test_generator_r.n // test_generator_r.batch_size) test_loss = np.sum(np.asarray(test_loss)) * 100 / ( test_generator_r.n // test_generator_r.batch_size) print('test_acc: {0} \t test_loss: {1}'.format(test_acc, test_loss)) batch_num += 1 # ********************************************************************************************** # learning rate schedule update: if learning is done using a single learning give the batch_num below a # high value if (batch_num == 3) | (batch_num == 5) | (batch_num == 7): lr = 0.1 * lr K.set_value(optimizer.lr, lr) print(K.get_value(optimizer.lr)) # ************************************************************************************************ # ----------------------------------------------------------------------------------------------- end_time = time.time() - start_time print("Total time taken %f :" % end_time) model_final.save_weights( '/home/yaurehman2/Documents/stereo_face_liveness/stereo_ckpt/Conventional/' + 'dual_grayscale_input_revised_protocol_3_' + str(args.max_epochs) + '.h5')
xtrain = np.array(xlist[:numTrainers]) ytrain = np.array(ylist[:numTrainers]) xtest = np.array(xlist[numTrainers:]) ytest = np.array(ylist[numTrainers:]) # Create the model model = Sequential() model.add(Dense(64, activation='relu', input_shape=( featureVec_size, ))) #small dataset - less hidden layers needed model.add(Dense(40, activation='relu')) model.add(Dense(final_dim, activation='sigmoid')) model.summary() # Stochastic Gradient Descent sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) history = model.fit(xtrain, ytrain, epochs=200, batch_size=batch_sz) # Time to test! score = model.evaluate(xtest, ytest, batch_size=batch_sz) print "\nCompiled! Here are your results..." print('Test loss:', score[0]) print('Test accuracy:', score[1]) all_featurevecs = xlist all_labels = ylist summaries = cross_validate(model, all_featurevecs, all_labels, 3)
return_sequences=True))(layer_b) decoder = dot([layer_a, layer_b], axes=1, normalize=True) #dense_1 = Dense(2048, activation='relu')(pool_2) #dense_2 = Dense(1024, activation='relu')(dense_1) conv_1 = Conv1D(128, word_length)(decoder) pool_1 = MaxPooling1D(5)(conv_1) drop = Dropout(0.5)(pool_1) global_pool = GlobalMaxPooling1D()(drop) dense_3 = Dense(512, activation='relu')(global_pool) output = Dense(10, activation='sigmoid')(dense_3) model = Model(inputs=[encoder_a, encoder_b], outputs=output) adam = Adam(lr=learning_rate) sgd = SGD(lr=learning_rate, nesterov=True, decay=1e-6, momentum=0.9) model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) print('Training shapes:', x_train.shape, y_train.shape) print('Valid shapes:', x_valid.shape, y_valid.shape) print(model.summary()) history = model.fit_generator( generate_batch(x_train, y_train, tokenizer), steps_per_epoch=steps_per_epoch, epochs=1, #len(x_train)//batch_size//steps_per_epoch, validation_data=generate_batch(x_valid, y_valid, tokenizer), validation_steps=steps_per_epoch) # Save the weights model.save_weights('model_weights.h5')
#SGD is an optimizer that uses a fixed learning rate #Start by using a function that creates a new model #create models in a for loops #each time we loop through, we compile the model using SGD with a different learning rate #We have created a function get_new_model() that creates an unoptimized model to optimize. from keras.optimizers import SGD def get_new_model(input_shape=input_shape): model = Sequential() model.add(Dense(100, activation='relu', input_shape=input_shape)) model.add(Dense(100, activation='relu')) model.add(Dense(2, activation='softmax')) return (model) #Create a list of learning rates lr_to_test = [.000001, 0.01, 1] #Loop over the learning rates for lr in lr_to_test: #Build new model to test, unaffected by previous models model = get_new_model() #Create SGD optimizer with specified learning rate: my_optimizer my_optimizer = SGD(lr=lr) #Compile the model model.compile(optimizer=my_optimizer, loss='categorical_crossentropy') #Fit the model model.fit(predictors, target)
from possum.datagen import datagen from keras.optimizers import SGD from keras.utils import to_categorical, plot_model from keras.callbacks import ModelCheckpoint, CSVLogger import numpy as np layers = 3 logger = CSVLogger("../log/r{:d}layer.log".format(layers)) X_valid = np.load("../data/valid/X.npy") y_valid = to_categorical(np.load("../data/valid/label.npy")) nu = ASKAP12() generator = datagen(nu, batch=64) sgd = SGD(lr=0.005, decay=0.001, momentum=0.9, nesterov=True) cnn = classifier(width=201) for i in range(2): cnn.conv(filters=64) cnn.batchNorm() cnn.activation('relu') cnn.pool(max_pool=True) for i in range(layers): cnn.residual(conv=3, activation='relu', batchNorm=True) cnn.globalPooling(max_pool=True) cnn.dropout(0.3) cnn.compile(optimizer=sgd) plot_model(cnn.model, 'residual.png')
def run_optimizer(n, iters): current_pop, current_pars = generate_custompop(n) score_list = [] print("Iteration 1") for i in range(n): print("Population " + str(i + 1)) #evaluate cnn_n = train(current_pop[i]) scores = cnn_n.evaluate(x_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1] * 100)) score_list.append(scores[1] * 100) m = max(score_list) ind = score_list.index(m) for ite in range(iters - 1): print("Iteration " + str(ite + 1)) current_pop, current_pars = get_next_generation(current_pars[ind], n) score_list = [] for i in range(n): #evaluate print("Population " + str(i + 1)) cnn_n = train(current_pop[i]) scores = cnn_n.evaluate(x_test, y_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1] * 100)) score_list.append(scores[1] * 100) m = max(score_list) ind = score_list.index(m) print('\n.\n.\n.\n.\nGetting best model') cnn_n = train(current_pop[ind]) sgd = SGD(lr=current_pars[ind][0], decay=current_pars[ind][1], momentum=current_pars[ind][0], nesterov=True) #save and retrieve with open("/home/aswin/proj/assets/model_num.txt", 'r') as f: model_num = int(f.read()) model_json = cnn_n.to_json() with open( "/home/aswin/proj/assets/models/model" + str(model_num) + ".json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 cnn_n.save_weights("/home/aswin/proj/assets/model" + str(model_num) + ".h5") print("Saved model to disk") # load json and create model json_file = open( "/home/aswin/proj/assets/models/model" + str(model_num) + ".json", 'r') loaded_model_json = json_file.read() json_file.close() loaded_model = model_from_json(loaded_model_json) # load weights into new model loaded_model.load_weights("/home/aswin/proj/assets/model" + str(model_num) + ".h5") print("Loaded model from disk") # evaluate loaded model on test data loaded_model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) score = loaded_model.evaluate(x_test, y_test, verbose=0) print("%s: %.2f%%" % (loaded_model.metrics_names[1], score[1] * 100)) with open("/home/aswin/proj/assets/model_num.txt", 'r+') as f: f.truncate(0) f.write(str(model_num + 1))
import numpy as np from keras.datasets import mnist from keras.models import Sequential from keras.layers.core import Dense, Activation from keras.optimizers import SGD from keras.utils import np_utils from make_tensorboard import make_tensorboard np.random.seed(1671) # for reproducibility # network and training NB_EPOCH = 20 BATCH_SIZE = 128 VERBOSE = 1 NB_CLASSES = 10 # number of outputs = number of digits OPTIMIZER = SGD() # optimizer, explained later in this chapter N_HIDDEN = 128 # 隠れ層のニューロン数 VALIDATION_SPLIT = 0.2 # how much TRAIN is reserved for VALIDATION # data: shuffled and split between train and test sets (X_train, y_train), (X_test, y_test) = mnist.load_data() # X_train is 60000 rows of 28x28 values --> reshaped in 60000 x 784 RESHAPED = 784 # X_train = X_train.reshape(60000, RESHAPED) X_test = X_test.reshape(10000, RESHAPED) X_train = X_train.astype('float32') X_test = X_test.astype('float32') # normalize
def __init__(self): # Variables to hold the description of the experiment self.config_description = "This is the template config file." # System dependent variable self._workers = 5 self._multiprocessing = True # Variables for comet.ml self._project_name = "jpeg_deep" self._workspace = "ssd" # Network variables self._weights = "/dlocal/home/2017018/bdegue01/weights/jpeg_deep/classification_dct/resnet_deconv/classification_dct_jpeg-deep_GYftBmXMdjdxoMksyI3e9VqB5IriBC9T/checkpoints/epoch-87_loss-0.7459_val_loss-1.5599.h5" self._network = SSD300_resnet(backbone="deconv_rfa", dct=True, image_shape=(38, 38)) # Training variables self._epochs = 240 self._batch_size = 32 self._steps_per_epoch = 1000 self.optimizer_parameters = {"lr": 0.001, "momentum": 0.9} self._optimizer = SGD(**self.optimizer_parameters) self._loss = SSDLoss(neg_pos_ratio=3, alpha=1.0).compute_loss self._metrics = None dataset_path = environ["DATASET_PATH"] images_2007_path = join(dataset_path, "VOC2007/JPEGImages") self.train_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/train.txt"))] self.validation_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/val.txt"))] self.test_sets = [(images_2007_path, join(dataset_path, "VOC2007/ImageSets/Main/test.txt"))] # Keras stuff self.model_checkpoint = None self.reduce_lr_on_plateau = ReduceLROnPlateau(patience=5, verbose=1) self.terminate_on_nan = TerminateOnNaN() self.early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=15) self._callbacks = [ self.reduce_lr_on_plateau, self.early_stopping, self.terminate_on_nan ] self.input_encoder = SSDInputEncoder() self.train_tranformations = [SSDDataAugmentation()] self.validation_transformations = [ ConvertTo3Channels(), Resize(height=300, width=300) ] self.test_transformations = [ ConvertTo3Channels(), Resize(height=300, width=300) ] self._train_generator = None self._validation_generator = None self._test_generator = None self._horovod = None
def learn_embedding(self, graph=None, edge_f=None, is_weighted=False, no_python=False): if not graph and not edge_f: raise Exception('graph/edge_f needed') if not graph: graph = graph_util.loadGraphFromEdgeListTxt(edge_f) if self._subsample: S = graph_util.randwalk_DiGraph_to_adj(graph, node_frac=self._node_frac, n_walks_per_node=self._n_walks_per_node, len_rw=self._len_rw) else: S = graph_util.transform_DiGraph_to_adj(graph) if not np.allclose(S.T, S): print "SDNE only works for symmetric graphs! Making the graph symmetric" t1 = time() S = (S + S.T)/2 # enforce S is symmetric S -= np.diag(np.diag(S)) # enforce diagonal = 0 self._node_num = S.shape[0] n_edges = np.count_nonzero(S) # Double counting symmetric edges deliberately to maintain autoencoder symmetry # Create matrix B B = np.ones(S.shape) B[S != 0] = self._beta # compute degree of each node deg = np.sum(S!=0, 1) # Generate encoder, decoder and autoencoder self._num_iter = self._n_iter # If cannot use previous step information, initialize new models self._encoder = get_encoder(self._node_num, self._d, self._K, self._n_units, self._nu1, self._nu2, self._actfn) self._decoder = get_decoder(self._node_num, self._d, self._K, self._n_units, self._nu1, self._nu2, self._actfn) self._autoencoder = get_autoencoder(self._encoder, self._decoder) # Initialize self._model # Input x_in = Input(shape=(2*self._node_num,), name='x_in') x1 = Lambda(lambda x: x[:,0:self._node_num], output_shape=(self._node_num,))(x_in) x2 = Lambda(lambda x: x[:,self._node_num:2*self._node_num], output_shape=(self._node_num,))(x_in) # Process inputs [x_hat1, y1] = self._autoencoder(x1) [x_hat2, y2] = self._autoencoder(x2) # Outputs x_diff1 = merge([x_hat1, x1], mode=lambda (a,b): a - b, output_shape=lambda L: L[1]) x_diff2 = merge([x_hat2, x2], mode=lambda (a,b): a - b, output_shape=lambda L: L[1]) y_diff = merge([y2, y1], mode=lambda (a,b): a - b, output_shape=lambda L: L[1]) # Objectives def weighted_mse_x(y_true, y_pred): ''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them. y_pred: Contains x_hat - x y_true: Contains [b, deg] ''' return KBack.sum(KBack.square(y_pred * y_true[:,0:self._node_num]), axis=-1)/y_true[:,self._node_num] def weighted_mse_y(y_true, y_pred): ''' Hack: This fn doesn't accept additional arguments. We use y_true to pass them. y_pred: Contains y2 - y1 y_true: Contains s12 ''' min_batch_size = KBack.shape(y_true)[0] return KBack.reshape(KBack.sum(KBack.square(y_pred), axis=-1),[min_batch_size, 1]) * y_true # Model self._model = Model(input=x_in, output=[x_diff1, x_diff2, y_diff]) sgd = SGD(lr=self._xeta, decay=1e-5, momentum=0.99, nesterov=True) # adam = Adam(lr=self._xeta, beta_1=0.9, beta_2=0.999, epsilon=1e-08) self._model.compile(optimizer=sgd, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha]) # self._model.compile(optimizer=adam, loss=[weighted_mse_x, weighted_mse_x, weighted_mse_y], loss_weights=[1, 1, self._alpha]) # Structure data in the correct format for the SDNE model # InData format: [x1, x2] # OutData format: [b1, b2, s12, deg1, deg2] data_chunk_size = 100000 InData = np.zeros((data_chunk_size, 2*self._node_num)) OutData = np.zeros((data_chunk_size, 2*self._node_num + 3)) # Train the model for epoch_num in range(self._num_iter): print 'EPOCH %d/%d' % (epoch_num, self._num_iter) e = 0 k = 0 for i in range(self._node_num): for j in range(self._node_num): if(S[i,j] != 0): temp = np.append(S[i,:], S[j,:]) InData[k,:] = temp temp = np.append(np.append(np.append(np.append(B[i,:], B[j,:]), S[i,j]), deg[i]), deg[j]) OutData[k,:] = temp e += 1 k += 1 if k == data_chunk_size: self._model.fit(InData, [ np.append(OutData[:,0:self._node_num], np.reshape(OutData[:,2*self._node_num+1], [data_chunk_size, 1]), axis=1), np.append(OutData[:,self._node_num:2*self._node_num], np.reshape(OutData[:,2*self._node_num+2], [data_chunk_size, 1]), axis=1), OutData[:,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1) k = 0 if k > 0: self._model.fit(InData[:k, :], [ np.append(OutData[:k,0:self._node_num], np.reshape(OutData[:k,2*self._node_num+1], [k, 1]), axis=1), np.append(OutData[:k,self._node_num:2*self._node_num], np.reshape(OutData[:k,2*self._node_num+2], [k, 1]), axis=1), OutData[:k,2*self._node_num] ], nb_epoch=1, batch_size=self._n_batch, shuffle=True, verbose=1) # Get embedding for all points _, self._Y = self._autoencoder.predict(S, batch_size=self._n_batch) t2 = time() # Save the autoencoder and its weights if(self._weightfile is not None): saveweights(self._encoder, self._weightfile[0]) saveweights(self._decoder, self._weightfile[1]) if(self._modelfile is not None): savemodel(self._encoder, self._modelfile[0]) savemodel(self._decoder, self._modelfile[1]) if(self._savefilesuffix is not None): saveweights(self._encoder, 'encoder_weights_'+self._savefilesuffix+'.hdf5') saveweights(self._decoder, 'decoder_weights_'+self._savefilesuffix+'.hdf5') savemodel(self._encoder, 'encoder_model_'+self._savefilesuffix+'.json') savemodel(self._decoder, 'decoder_model_'+self._savefilesuffix+'.json') # Save the embedding np.savetxt('embedding_'+self._savefilesuffix+'.txt', self._Y) return self._Y, (t2-t1)
def cross_validate_inmemory(model_name, **kwargs): """ StateFarm competition: Training set has 26 unique drivers. We do 26 fold CV where a driver is alternatively singled out to be the validation set Load the whole train data in memory for faster operations args: model (keras model) **kwargs (dict) keyword arguments that specify the model hyperparameters """ # Roll out the parameters nb_classes = kwargs["nb_classes"] batch_size = kwargs["batch_size"] n_batch_per_epoch = kwargs["n_batch_per_epoch"] nb_epoch = kwargs["nb_epoch"] prob = kwargs["prob"] do_plot = kwargs["do_plot"] data_file = kwargs["data_file"] semi_super_file = kwargs["semi_super_file"] pretr_weights_file = kwargs["pretr_weights_file"] normalisation_style = kwargs["normalisation_style"] weak_labels = kwargs["weak_labels"] objective = kwargs["objective"] experiment = kwargs["experiment"] start_fold = kwargs["start_fold"] # Load env variables in (in .env file at the root of the project) load_dotenv(find_dotenv()) # Load env variables model_dir = os.path.expanduser(os.environ.get("MODEL_DIR")) data_dir = os.path.expanduser(os.environ.get("DATA_DIR")) # Output path where we store experiment log and weights model_dir = os.path.join(model_dir, model_name) # Create if it does not exist general_utils.create_dir(model_dir) # Automatically determine experiment name list_exp = glob.glob(model_dir + "/*") # Create the experiment dir and weights dir if experiment: exp_dir = os.path.join(model_dir, experiment) else: exp_dir = os.path.join(model_dir, "Experiment_%s" % len(list_exp)) general_utils.create_dir(exp_dir) # Compile model. # opt = RMSprop(lr=5E-6, rho=0.9, epsilon=1e-06) opt = SGD(lr=5e-4, decay=1e-6, momentum=0.9, nesterov=True) # opt = Adam(lr=1E-5, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # Batch generator DataAug = batch_utils.AugDataGenerator(data_file, batch_size=batch_size, prob=prob, dset="train", maxproc=4, num_cached=60, random_augm=False, hdf5_file_semi=semi_super_file) DataAug.add_transform("h_flip") # DataAug.add_transform("v_flip") # DataAug.add_transform("fixed_rot", angle=40) DataAug.add_transform("random_rot", angle=40) # DataAug.add_transform("fixed_tr", tr_x=40, tr_y=40) DataAug.add_transform("random_tr", tr_x=40, tr_y=40) # DataAug.add_transform("fixed_blur", kernel_size=5) DataAug.add_transform("random_blur", kernel_size=5) # DataAug.add_transform("fixed_erode", kernel_size=4) DataAug.add_transform("random_erode", kernel_size=3) # DataAug.add_transform("fixed_dilate", kernel_size=4) DataAug.add_transform("random_dilate", kernel_size=3) # DataAug.add_transform("fixed_crop", pos_x=10, pos_y=10, crop_size_x=200, crop_size_y=200) DataAug.add_transform("random_crop", min_crop_size=140, max_crop_size=160) # DataAug.add_transform("hist_equal") # DataAug.add_transform("random_occlusion", occ_size_x=100, occ_size_y=100) epoch_size = n_batch_per_epoch * batch_size general_utils.pretty_print("Load all data...") with h5py.File(data_file, "r") as hf: X = hf["train_data"][:, :, :, :] y = hf["train_label"][:].astype(np.uint8) y = np_utils.to_categorical(y, nb_classes=nb_classes) # Format for keras try: for fold in range(start_fold, 8): # for fold in np.random.permutation(26): min_valid_loss = 100 # Save losses list_train_loss = [] list_valid_loss = [] # Load valid data in memory for fast error evaluation idx_valid = hf["valid_fold%s" % fold][:] idx_train = hf["train_fold%s" % fold][:] X_valid = X[idx_valid] y_valid = y[idx_valid] # Normalise X_valid = normalisation(X_valid, normalisation_style) # Compile model general_utils.pretty_print("Compiling...") model = models.load(model_name, nb_classes, X_valid.shape[-3:], pretr_weights_file=pretr_weights_file) model.compile(optimizer=opt, loss=objective) # Save architecture json_string = model.to_json() with open(os.path.join(data_dir, '%s_archi.json' % model.name), 'w') as f: f.write(json_string) for e in range(nb_epoch): # Initialize progbar and batch counter progbar = generic_utils.Progbar(epoch_size) batch_counter = 1 l_train_loss = [] start = time.time() for X_train, y_train in DataAug.gen_batch_inmemory(X, y, idx_train=idx_train): if do_plot: general_utils.plot_batch(X_train, np.argmax(y_train, 1), batch_size) # Normalise X_train = normalisation(X_train, normalisation_style) train_loss = model.train_on_batch(X_train, y_train) l_train_loss.append(train_loss) batch_counter += 1 progbar.add(batch_size, values=[("train loss", train_loss)]) if batch_counter >= n_batch_per_epoch: break print("") print('Epoch %s/%s, Time: %s' % (e + 1, nb_epoch, time.time() - start)) y_valid_pred = model.predict(X_valid, verbose=0, batch_size=16) train_loss = float(np.mean(l_train_loss)) # use float to make it json saveable valid_loss = log_loss(y_valid, y_valid_pred) print("Train loss:", train_loss, "valid loss:", valid_loss) list_train_loss.append(train_loss) list_valid_loss.append(valid_loss) # Record experimental data in a dict d_log = {} d_log["fold"] = fold d_log["nb_classes"] = nb_classes d_log["batch_size"] = batch_size d_log["n_batch_per_epoch"] = n_batch_per_epoch d_log["nb_epoch"] = nb_epoch d_log["epoch_size"] = epoch_size d_log["prob"] = prob d_log["optimizer"] = opt.get_config() d_log["augmentator_config"] = DataAug.get_config() d_log["train_loss"] = list_train_loss d_log["valid_loss"] = list_valid_loss json_file = os.path.join(exp_dir, 'experiment_log_fold%s.json' % fold) general_utils.save_exp_log(json_file, d_log) # Only save the best epoch if valid_loss < min_valid_loss: min_valid_loss = valid_loss trained_weights_path = os.path.join(exp_dir, '%s_weights_fold%s.h5' % (model.name, fold)) model.save_weights(trained_weights_path, overwrite=True) except KeyboardInterrupt: pass
from sklearn.preprocessing import normalize # from sklearn.model_selection import StratifiedKFold, cross_val_score from itertools import combinations_with_replacement, product from datetime import datetime from configparser import ConfigParser import matplotlib.pyplot as plt os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' reading_index = slice(None, None, None) optimizer_dict = { 'adam': Adam(), 'nadam': Nadam(), 'sgd': SGD(lr=0.0000001, decay=0, momentum=0.6, nesterov=True), 'adadelta': Adadelta(clipnorm=1.) } # seed for reproducibility np.random.seed(1337) pair_slice_choices = [[2], [2, 6], [0, 2, 4, 6], slice(None, None, None)] triplet_slice_choices = [[8], [8, 9], [8, 9, 26, 27], [0, 1, 8, 9, 18, 19, 26, 27], [ 0, 1, 4, 5, 8, 9, 12, 13, 18, 19, 22, 23, 26, 27, 30, 31 ], slice(None, None, None)]