def adagrad_optimizer(self, learning_rate): optimizer = optimizers.Adagrad(learning_rate=learning_rate) return optimizer
def lstm_dense_sunspots(args): """ Main function """ # %% # IMPORTS # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \ series_to_supervised, affine_transformation from artificial_neural_networks.code.utils.vis_utils import regression_figs # %% if args.verbose > 0: print(args) # For reproducibility if args.reproducible: os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) tf.set_random_seed(args.seed) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) # print(hash("keras")) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back train = sunspots[:n_split] test = sunspots[n_split - look_back:] train_x, train_y = series_to_supervised(train, look_back) test_x, test_y = series_to_supervised(test, look_back) # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation n_train = train_x.shape[0] # number of training examples/samples n_test = test_x.shape[0] # number of test examples/samples n_in = train_x.shape[1] # number of features / dimensions n_out = train_y.shape[1] # number of steps ahead to be predicted # Reshape training and test sets train_x = train_x.reshape(n_train, n_in, 1) test_x = test_x.reshape(n_test, n_in, 1) # Apply preprocessing train_x_ = affine_transformation(train_x, scaling_factor, translation) train_y_ = affine_transformation(train_y, scaling_factor, translation) test_x_ = affine_transformation(test_x, scaling_factor, translation) test_y_ = affine_transformation(test_y, scaling_factor, translation) # %% # Model hyperparameters and ANN Architecture x = Input(shape=(n_in, 1)) # input layer h = x h = LSTM(units=args.layer_size)(h) # hidden layer out = Dense(units=n_out, activation=None)(h) # output layer model = Model(inputs=x, outputs=out) if args.verbose > 0: model.summary() def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) loss_function = root_mean_squared_error metrics = ['mean_absolute_error', 'mean_absolute_percentage_error'] lr = args.lrearning_rate epsilon = args.epsilon optimizer_selection = { 'Adadelta': optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0), 'Adagrad': optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0), 'Adam': optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False), 'Adamax': optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0), 'Nadam': optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004), 'RMSprop': optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0), 'SGD': optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False) } optimizer = optimizer_selection[args.optimizer] model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics) # %% # Save trained models for every epoch models_path = r'artificial_neural_networks/trained_models/' model_name = 'sunspots_lstm_dense' weights_path = models_path + model_name + '_weights' model_path = models_path + model_name + '_model' file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}' if args.save_weights_only: file_path = weights_path else: file_path = model_path file_path += file_suffix monitor = 'val_loss' if args.save_models: checkpoint = ModelCheckpoint(file_path + '.h5', monitor=monitor, verbose=args.verbose, save_best_only=args.save_best, mode='auto', save_weights_only=args.save_weights_only) callbacks = [checkpoint] else: callbacks = [] # %% # TRAINING PHASE if args.time_training: start = timer() model.fit(x=train_x_, y=train_y_, validation_data=(test_x_, test_y_), batch_size=args.batch_size, epochs=args.n_epochs, verbose=args.verbose, callbacks=callbacks) if args.time_training: end = timer() duration = end - start print('Total time for training (in seconds):') print(duration) # %% # TESTING PHASE # Predict preprocessed values train_y_pred_ = model.predict(train_x_)[:, 0] test_y_pred_ = model.predict(test_x_)[:, 0] # Remove preprocessing train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True) test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True) train_rmse = sqrt(mean_squared_error(train_y, train_y_pred)) train_mae = mean_absolute_error(train_y, train_y_pred) train_r2 = r2_score(train_y, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y, test_y_pred)) test_mae = mean_absolute_error(test_y, test_y_pred) test_r2 = r2_score(test_y, test_y_pred) if args.verbose > 0: print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if args.plot: regression_figs(train_y=train_y, train_y_pred=train_y_pred, test_y=test_y, test_y_pred=test_y_pred) # %% # Save the architecture and the lastly trained model save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix, test_rmse, test_mae, args) # %% return model
model.add(Conv2D(64, (3, 3), padding='same')) model.add(Activation('relu')) model.add(Conv2D(64, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.25)) model.add(Flatten()) model.add(Dense(512)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(1)) model.add(Activation('softmax')) ''' sgd = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0) model.compile(loss='mean_squared_error', optimizer= sgd, metrics=['accuracy']) # this is the augmentation configuration we will use for training train_datagen = ImageDataGenerator( rescale=1. / 255, shear_range=0.2, zoom_range=0.2, horizontal_flip=True)
import tensorflow as tf from keras import optimizers from keras.layers import Input from keras.models import Model from keras.layers import Dense, Flatten, Reshape, Dropout from keras.layers import Convolution1D, MaxPooling1D, BatchNormalization from keras.layers import Lambda def mat_mul(A, B): return tf.matmul(A, B) # adam = optimizers.Adam(lr=0.1, decay= 0.1/epoch_num) adam = optimizers.Adagrad(lr= learning_rate ,decay= 0) #0.01 , 0.7 worked well # ------------------------------------ Pointnet Architecture # input_Transformation_net input_points = Input(shape=(num_points, 3)) x = Convolution1D(64, 1, activation='relu', input_shape=(num_points, 3))(input_points) x = BatchNormalization()(x) x = Convolution1D(128, 1, activation='relu')(x) x = BatchNormalization()(x) x = Convolution1D(1024, 1, activation='relu')(x) x = BatchNormalization()(x) x = MaxPooling1D(pool_size=num_points)(x) x = Dense(512, activation='relu')(x) x = BatchNormalization()(x) x = Dense(256, activation='relu')(x)
model.add(Dense(256)) model.add(Activation('sigmoid')) model.add(Dropout(0.5)) model.add(Dense(16)) model.add(Activation('sigmoid')) model.add(Dropout(0.5)) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizers.Adagrad(lr=0.0058), metrics=['accuracy']) model.load_weights("Models/Model1/weights.h5") data = generate_arrays(data_dir) res = model.predict_classes(data[0], verbose=0) print("Total: ", len(res)) # true pos, true neg, false pos, false neg for i in range(len(res)): if res[i] == 1 and data[1][i] == 1: tp += 1 elif res[i] == 1 and data[1][i] == 0: fp += 1 elif res[i] == 0 and data[1][i] == 1:
def train_VGG19_Model(csv_file , lr , ep): def step_decay_schedule(initial_lr=1e-3, decay_factor=0.75, step_size=10): def schedule(epoch): return initial_lr * (decay_factor ** np.floor(epoch / step_size)) return LearningRateScheduler(schedule) def Read_image(path): im = Image.open(path).convert('RGB') return im X = [] Y = [] dataset = pd.read_csv(csv_file) for index, row in dataset.iterrows(): X.append(array(Read_image(row[0]).resize((100, 100))).flatten() / 255.0) Y.append(row[1]) X = np.array(X) Y = to_categorical(Y, 2) X = X.reshape(-1, 100, 100, 3) X_train, X_val, Y_train, Y_val = train_test_split(X, Y, test_size=0.20, random_state=5) # Load the VGG model vgg_conv = VGG19(weights='imagenet', include_top=False, input_shape=(100, 100, 3)) # Create the model model = models.Sequential() # Freeze the layers the first layers for layer in vgg_conv.layers[:-5]: layer.trainable = False # Check the trainabl status of the individual layers for layer in vgg_conv.layers: print(layer, layer.trainable) model.add(vgg_conv) model.summary() model.add(layers.Flatten()) model.add(layers.Dense(1024, activation='relu')) model.add(layers.Dropout(0.50)) model.add(layers.Dense(1024, activation='relu')) model.add(layers.Dropout(0.50)) model.add(layers.Dense(2, activation='softmax')) optimizer = optimizers.Adagrad(lr=lr, epsilon=None, decay=0.0) model.compile(optimizer=optimizer, loss="mean_squared_error", metrics=["accuracy"]) lr_sched = step_decay_schedule(initial_lr=1e-4, decay_factor=0.75, step_size=2) epochs = ep batch_size = 20 history = model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_val, Y_val), verbose=2,callbacks=[lr_sched]) # Plot the loss and accuracy curves for training and validation fig, ax = plt.subplots(3, 1) ax[0].plot(history.history['loss'], color='b', label="Training loss") ax[0].plot(history.history['val_loss'], color='r', label="validation loss", axes=ax[0]) legend = ax[0].legend(loc='best', shadow=True) ax[1].plot(history.history['acc'], color='b', label="Training accuracy") ax[1].plot(history.history['val_acc'], color='r', label="Validation accuracy") legend = ax[1].legend(loc='best', shadow=True) def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): plt.imshow(cm, interpolation='nearest', cmap=cmap) plt.title(title) plt.colorbar() tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, rotation=45) plt.yticks(tick_marks, classes) if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] thresh = cm.max() / 2. for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): plt.text(j, i, cm[i, j], horizontalalignment="center", color="white" if cm[i, j] > thresh else "black") plt.tight_layout() plt.ylabel('True label') plt.xlabel('Predicted label') # Predict the values from the validation dataset Y_pred = model.predict(X_val) # Convert predictions classes to one hot vectors Y_pred_classes = np.argmax(Y_pred, axis=1) # Convert validation observations to one hot vectors Y_true = np.argmax(Y_val, axis=1) # compute the confusion matrix confusion_mtx = confusion_matrix(Y_true, Y_pred_classes) # plot the confusion matrix plot_confusion_matrix(confusion_mtx, classes=range(2)) image_path = os.getcwd()+"\\Figures" Models_path = os.getcwd()+"\\Re_Traind_Models" file_number =random.randint(1, 1000000) plot_Name = image_path+"\\VGG19_"+str(file_number)+".png" Model_Name = Models_path+"\\VGG19_"+str(file_number)+".h5" plt.savefig(plot_Name , transparent =True , bbox_incehs="tight" , pad_inches = 2 , dpi = 50) model.save(Model_Name) return plot_Name , Model_Name
def run_nn(**kwargs): """ Run the neural network for the given parameters Adapted from the code provided in lecture """ # Start the timer start_t = timer() # Number of input, hidden, and output nodes input_nodes = 784 hidden_nodes = 200 output_nodes = 10 # Set parameters learning_rate = kwargs["learning_rate"] optimizer = kwargs["optimizer"] batch_size = kwargs["batch_size"] epochs = kwargs["epochs"] # Create the Keras model model = Sequential() model.add( Dense( hidden_nodes, activation='sigmoid', input_shape=(input_nodes, ), bias=False, )) model.add(Dense( output_nodes, activation='sigmoid', bias=False, )) # Print the model summary model.summary() # Set the optimizer if optimizer == "adam": opt = optimizers.Adam(learning_rate=learning_rate) elif optimizer == "sgd": opt = optimizers.SGD(learning_rate=learning_rate) elif optimizer == "rmsprop": opt = optimizers.RMSprop(learning_rate=learning_rate) elif optimizer == "adagrad": opt = optimizers.Adagrad(learning_rate=learning_rate) elif optimizer == "adadelta": opt = optimizers.Adadelta(learning_rate=learning_rate) elif optimizer == "adamax": opt = optimizers.Adamax(learning_rate=learning_rate) elif optimizer == "nadam": opt = optimizers.Nadam(learning_rate=learning_rate) # Default optimizer is adam else: opt = optimizers.Adam(learning_rate=learning_rate) # Define the error criterion, optimizer, and an optional metric to monitor during training model.compile( loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'], ) # Load the mnist training data CSV df = pd.read_csv("mnist_csv/mnist_train.csv", header=None) # Columns 1-784 are the input values x_train = np.asfarray(df.loc[:, 1:input_nodes].values) x_train /= 255.0 # Column 0 is the desired label labels = df.loc[:, 0].values # Convert labels to one-hot vectors y_train = np_utils.to_categorical(labels, output_nodes) # Train the neural network # Train the model model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1) # Save the model model.save('MNIST_3layer_keras.h5') print('model saved') # Test the model # Load the MNIST test data CSV file into a list test_data_file = open('mnist_csv/mnist_test.csv', 'r') test_data_list = test_data_file.readlines() test_data_file.close() # Scorecard for how well the network performs, initially empty scorecard = [] # Go through all the data in the test data set, one by one for record in test_data_list: # Split the record by the commas data_sample = record.split(',') # Correct answer is first value correct_label = int(data_sample[0]) # Scale and shift the inputs inputs = np.asfarray(data_sample[1:]) / 255.0 # Make prediction outputs = model.predict(np.reshape(inputs, (1, len(inputs)))) # The index of the highest value corresponds to the label label = np.argmax(outputs) # Append correct or incorrect to list if label == correct_label: # Network's answer matches correct answer, add 1 to scorecard scorecard.append(1) else: # Netowrk's answer doesn't match correct answer, add 0 to scorecard scorecard.append(0) pass pass # Calculate the accuracy scorecard_array = np.asarray(scorecard) accuracy = scorecard_array.sum() / scorecard_array.size print('accuracy = {}'.format(accuracy)) # Stop the timer end_t = timer() execution_time = end_t - start_t print('elapsed time = {}'.format(execution_time)) output = {'accuracy': accuracy, 'execution_time': execution_time} return output
model.add(Dense(512)) model.add(BatchNormalization()) model.add(Activation('relu')) model.add(Dropout(0.75)) model.add(Dense(64)) model.add(Activation('sigmoid')) model.add(Dense(1)) model.add(Activation('sigmoid')) model.compile(loss='binary_crossentropy', optimizer= optimizers.Adagrad(lr=0.05), metrics=['accuracy']) def generate_arrays_from_dir(path, batchsz): db = pd.read_csv(db_file) while 1: with open(path) as f: r = csv.reader(f) batchCount = 0 batchX = [] batchy = [] for ln in r: X = np.array(list(np.array(ln[1:(len(ln))],dtype=np.uint32).tobytes())) y = 1 if db.loc[db['id'] == int(ln[0][1:])].values[0][2] == 'p' else 0 batchX.append(np.array(X))
def create_model(x_train, y_train, x_val, y_val, layer_sizes): def coeff_determination(y_true, y_pred): SS_res = K.sum(K.square(y_true - y_pred)) SS_tot = K.sum(K.square(y_true - K.mean(y_true))) return (1 - SS_res / (SS_tot + K.epsilon())) model = models.Sequential() model.add( layers.Dense( {{ choice([ np.power(2, 0), np.power(2, 1), np.power(2, 2), np.power(2, 3), np.power(2, 4), np.power(2, 5), np.power(2, 6), np.power(2, 7) ]) }}, activation={{ choice([ 'relu', 'selu', 'tanh', 'softmax', 'softplus', 'linear', None ]) }}, input_shape=(len(data.columns), ))) model.add( layers.Dense( {{ choice([ np.power(2, 0), np.power(2, 1), np.power(2, 2), np.power(2, 3), np.power(2, 4), np.power(2, 5), np.power(2, 6), np.power(2, 7) ]) }}, activation={{ choice([ 'relu', 'selu', 'tanh', 'softmax', 'softplus', 'linear', None ]) }})) model.add( layers.Dense( {{ choice([ np.power(2, 0), np.power(2, 1), np.power(2, 2), np.power(2, 3), np.power(2, 4), np.power(2, 5), np.power(2, 6), np.power(2, 7) ]) }}, activation={{ choice([ 'relu', 'selu', 'tanh', 'softmax', 'softplus', 'linear', None ]) }})) model.add( layers.Dense(1, activation={{ choice([ 'relu', 'selu', 'tanh', 'softmax', 'softplus', 'linear', None ]) }})) RMS = optimizers.Adagrad(lr={{choice([0.0001, 0.001, 0.01, 0.1])}}) model.compile(optimizer=RMS, loss={{ choice([ 'mean_absolute_error', 'mean_squared_error', 'mean_absolute_percentage_error', 'mean_squared_logarithmic_error', 'hinge', 'squared_hinge', 'logcosh' ]) }}, metrics=[coeff_determination]) model.fit(x_train, y_train, epochs={{choice([25, 50, 75, 100, 500])}}, batch_size={{choice([10, 16, 20, 32, 64])}}, validation_data=(x_val, y_val)) score, acc = model.evaluate(x_val, y_val, verbose=0) print('Validation accuracy:', acc) return {'loss': -acc, 'status': STATUS_OK, 'model': model}
#Train Conciseness Model cvscores_conciseness = [] for train, test in kfold.split(X_train, y_train): print("train", train) print("test", test) y_train_cat = keras.utils.to_categorical(y_train, 2) #Build keras model model = Sequential() model.add(Dense(units=100, activation='relu', input_dim=shape[1])) model.add(Dropout(rate=0.2)) model.add(Dense(units=50, activation='relu')) model.add(Dense(units=2, activation='softmax')) sgd = optimizers.Adagrad(lr=0.01) model.compile(loss='mean_squared_error', optimizer=sgd, metrics=['mse']) #Train model model.fit(X_train[train], y_train_cat[train], epochs=150, batch_size=128, validation_data=(X_train[test], y_train_cat[test])) loss_and_metrics = model.evaluate(X_train[test], y_train_cat[test], batch_size=128) print(loss_and_metrics)
def get_model(name, **kwargs): """ Instantiate and obtain a model with thr hyperparameters Args: name: string of the model name kwargs: hyperparameters Returns: model: Keras network optimizer: Keras optimizer criterion: Keras loss Function kwargs: hyperparameters with sane defaults """ # cuda = kwargs.setdefault('cuda', False) n_classes = kwargs['n_classes'] #name = kwargs['dataset'] n_bands = kwargs['n_bands'] weights = np.ones(n_classes) # weights[torch.LongTensor(kwargs['ignored_labels'])] = 0. weights = kwargs.setdefault('weights', weights) model_name = None if name == 'nn': kwargs.setdefault('patch_size', 1) center_pixel = True #model,model_name = Baseline(n_bands, n_classes, kwargs.setdefault('dropout', True)) model, model_name = _1April(n_bands, n_classes, kwargs.setdefault('dropout', True)) lr = kwargs.setdefault('learning_rate', 0.0001) optimizer = optimizers.Adam(lr=lr) # optimizer = optim.Adam(model.parameters(), lr=lr) criterion = 'categorical_crossentropy' kwargs.setdefault('epoch', 100) kwargs.setdefault('batch_size', 100) elif name == 'hu': kwargs.setdefault('patch_size', 1) kwargs.setdefault('epoch', 100) kwargs.setdefault('batch_size', 100) center_pixel = True # input_channels=((kwargs['batch_size'],n_bands)) model, model_name = HuEtAl.build(n_bands, n_classes) lr = kwargs.setdefault('learning_rate', 0.01) optimizer = optimizers.Adam(lr=lr) criterion = 'categorical_crossentropy' elif name == 'hamida': patch_size = kwargs.setdefault('patch_size', 5) center_pixel = True model, model_name = HamidaEtAl.build(n_bands, n_classes, patch_size=patch_size) lr = kwargs.setdefault('learning_rate', 0.01) optimizer = optimizers.SGD(lr=lr, decay=0.0005) kwargs.setdefault('batch_size', 100) criterion = 'categorical_crossentropy' elif name == 'lee': kwargs.setdefault('epoch', 200) patch_size = kwargs.setdefault('patch_size', 5) center_pixel = True model, model_name = LeeEtAl.build(n_bands, n_classes) lr = kwargs.setdefault('learning_rate', 0.001) optimizer = optimizers.Adam(lr=lr) criterion = 'categorical_crossentropy' elif name == 'chen': patch_size = kwargs.setdefault('patch_size', 27) center_pixel = True model, model_name = ChenEtAl.build(n_bands, n_classes, patch_size=patch_size) lr = kwargs.setdefault('learning_rate', 0.003) optimizer = optimizers.SGD(lr=lr) criterion = 'categorical_crossentropy' kwargs.setdefault('epoch', 400) kwargs.setdefault('batch_size', 100) elif name == 'li': patch_size = kwargs.setdefault('patch_size', 5) center_pixel = True model, model_name = LiEtAl.build(n_bands, n_classes, n_planes=16, patch_size=patch_size) lr = kwargs.setdefault('learning_rate', 0.01) optimizer = optimizers.SGD(lr=lr, momentum=0.9, decay=0.0005) epoch = kwargs.setdefault('epoch', 200) criterion = 'categorical_crossentropy' elif name == 'he': kwargs.setdefault('patch_size', 7) kwargs.setdefault('batch_size', 40) lr = kwargs.setdefault('learning_rate', 0.01) center_pixel = True model, model_name = HeEtAl.build(n_bands, n_classes, patch_size=kwargs['patch_size']) # For Adagrad, we need to load the model on GPU before creating the optimizer optimizer = optimizers.Adagrad(lr=lr, decay=0.01) criterion = 'categorical_crossentropy' elif name == 'luo': # All the experiments are settled by the learning rate of 0.1, # the decay term of 0.09 and batch size of 100. kwargs.setdefault('patch_size', 3) kwargs.setdefault('batch_size', 100) lr = kwargs.setdefault('learning_rate', 0.1) center_pixel = True model, model_name = LuoEtAl.build(n_bands, n_classes, patch_size=kwargs['patch_size']) optimizer = optimizers.SGD(lr=lr, decay=0.09) criterion = 'categorical_crossentropy' elif name == 'sharma': # We train our S-CNN from scratch using stochastic gradient descent with # momentum set to 0.9, weight decay of 0.0005, and with a batch size # of 60. We initialize an equal learning rate for all trainable layers # to 0.05, which is manually decreased by a factor of 10 when the validation # error stopped decreasing. Prior to the termination the learning rate was # reduced two times at 15th and 25th epoch. [...] # We trained the network for 30 epochs kwargs.setdefault('batch_size', 60) epoch = kwargs.setdefault('epoch', 30) lr = kwargs.setdefault('lr', 0.05) center_pixel = True # We assume patch_size = 64 kwargs.setdefault('patch_size', 64) model, model_name = SharmaEtAl.build(n_bands, n_classes, patch_size=kwargs['patch_size']) optimizer = optimizers.SGD(lr=lr, decay=0.0005) criterion = 'categorical_crossentropy' elif name == 'liu': kwargs['supervision'] = 'semi' # "The learning rate is set to 0.001 empirically. The number of epochs is set to be 40." kwargs.setdefault('epoch', 40) lr = kwargs.setdefault('lr', 0.001) center_pixel = True patch_size = kwargs.setdefault('patch_size', 9) model, model_name = LiuEtAl.build(n_bands, n_classes, patch_size) optimizer = optimizers.SGD(lr=lr) criterion = ['categorical_crossentropy', 'mean_squared_error'] #weighted_loss(1.0) # K.mean(K.square(rec, squeeze_all(data[:,:,:,patch_size//2,patch_size//2]))) # kwargs.setdefault('scheduler', optim.lr_scheduler.MultiStepLR(optimizer, milestones=[epoch // 2, (5 * epoch) // 6], gamma=0.1)) elif name == 'boulch': kwargs['supervision'] = 'semi' kwargs.setdefault('patch_size', 1) kwargs.setdefault('epoch', 100) lr = kwargs.setdefault('lr', 0.001) center_pixel = True model, model_name = BoulchEtAl.build(n_bands, n_classes) optimizer = optimizers.SGD(lr=lr) criterion = ['categorical_crossentropy', 'mean_squared_error'] #weighted_loss(0.1) elif name == 'mou': kwargs.setdefault('patch_size', 1) center_pixel = True kwargs.setdefault('epoch', 100) # "The RNN was trained with the Adadelta algorithm [...] We made use of a # fairly high learning rate of 1.0 instead of the relatively low # default of 0.002 to train the network" lr = kwargs.setdefault('lr', 1.0) model, model_name = MouEtAl.build(n_bands, n_classes) # For Adadelta, we need to load the model on GPU before creating the optimizer # model = model.to(device) optimizer = optimizers.Adadelta(lr=lr) criterion = 'categorical_crossentropy' elif name == 'squeezenet': kwargs.setdefault('patch_size', 3) kwargs.setdefault('batch_size', 40) kwargs.setdefault('epoch', 100) lr = kwargs.setdefault('learning_rate', 0.5) center_pixel = True model, model_name = Squeezenet().build(n_bands, n_classes, patch_size=kwargs['patch_size']) optimizer = optimizers.Adadelta(lr=lr) criterion = 'categorical_crossentropy' else: raise KeyError("{} model is unknown.".format(name)) epoch = kwargs.setdefault('epoch', 100) #kwargs.setdefault('scheduler', None) kwargs.setdefault('batch_size', 100) kwargs.setdefault('dataset', None) kwargs.setdefault('supervision', 'full') kwargs.setdefault('flip_augmentation', False) kwargs.setdefault('radiation_augmentation', False) kwargs.setdefault('mixture_augmentation', False) kwargs['center_pixel'] = center_pixel return model, model_name, optimizer, criterion, kwargs
# generate dataset train_choking_x, train_choking_y = data_loader.read("train_choking") vali_choking_x, vali_choking_y = data_loader.read("validation_choking") num, time, w, h, color = train_choking_x.shape model_recog = models.Sequential() model_recog.add(layers.Reshape((time, -1))) model_recog.add( layers.LSTM(units=64, dropout=0.1, recurrent_dropout=0.1, return_sequences=True)) model_recog.add(layers.LSTM(units=32, dropout=0.1, recurrent_dropout=0.1)) model_recog.add(layers.Dense(1, activation="sigmoid")) model_recog.compile(optimizer=optimizers.Adagrad(), loss='binary_crossentropy', metrics=['acc']) history = model_recog.fit(train_choking_x, train_choking_y, epochs=10, batch_size=30, validation_data=(vali_choking_x, vali_choking_y)) model_recog.save("choking_model.h5") #y_pred = model.predict(test_choking_x) #print("real_label", vali_choking_y) acc = history.history['acc'] val_acc = history.history['val_acc'] loss = history.history['loss']
def cnnClassifier(): img_height_rows = 32 img_width_cols = 32 im_shape = (img_height_rows, img_width_cols, 1) print(im_shape) x_train = X_train.reshape( X_train.shape[0], *im_shape) # Python TIP :the * operator unpacks the tuple x_test = X_test.reshape(X_test.shape[0], *im_shape) global cnn kernelSize = (3, 3) ip_activation = 'relu' ip_conv_0 = Conv2D(filters=4, kernel_size=kernelSize, input_shape=im_shape, activation=ip_activation) cnn.add(ip_conv_0) # Add the next Convolutional+Activation layer ip_conv_0_1 = Conv2D(filters=4, kernel_size=kernelSize, activation=ip_activation) cnn.add(ip_conv_0_1) # Add the Pooling layer pool_0 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding="same") cnn.add(pool_0) ip_conv_1 = Conv2D(filters=4, kernel_size=kernelSize, activation=ip_activation) cnn.add(ip_conv_1) ip_conv_1_1 = Conv2D(filters=4, kernel_size=kernelSize, activation=ip_activation) cnn.add(ip_conv_1_1) pool_1 = MaxPool2D(pool_size=(2, 2), strides=(2, 2), padding="same") cnn.add(pool_1) flat_layer_0 = Flatten() cnn.add(Flatten()) # Now add the Dense layers h_dense_0 = Dense(units=20, activation=ip_activation, kernel_initializer='uniform') cnn.add(h_dense_0) # Let's add one more before proceeding to the output layer h_dense_1 = Dense(units=1024, activation=ip_activation, kernel_initializer='uniform', name='dense11') cnn.add(h_dense_1) n_classes = 36 op_activation = 'softmax' output_layer = Dense(units=n_classes, activation=op_activation, kernel_initializer='uniform') cnn.add(output_layer) opt = optimizers.Adagrad(lr=0.001) loss = 'categorical_crossentropy' metrics = ['accuracy'] # Compile the classifier using the configuration we want cnn.compile(optimizer=opt, loss=loss, metrics=metrics) history = cnn.fit(x_train, T_train, batch_size=300, epochs=5, validation_data=(x_test, T_test)) scores = cnn.evaluate(x_test, T_test, verbose=0) print("Accuracy: %.2f%%" % (scores[1] * 100))
s.append(v) df[col] = (df[col] - m) / v scale.append(s) # In[7]: x_train = df.values y_train = Y # In[8]: model_1 = Sequential() model_1.add( Dense(len(x_train[0]), activation='relu', input_dim=len(x_train[0]))) model_1.add(Dense(1, activation='sigmoid')) model_1.compile(optimizer=optimizers.Adagrad(lr=0.0123), loss='binary_crossentropy', metrics=['binary_accuracy']) model_1.summary() # In[9]: history1 = model_1.fit(x_train, y_train, epochs=50, batch_size=128, validation_split=0.1) # In[10]: model_2 = Sequential()
""" # ---------------------------------------------- # Training #def top_6(y_true, y_pred): # return tf.keras.metrics.top_k_categorical_accuracy(y_true, y_pred, k=6) #from keras.utils import multi_gpu_model #model = multi_gpu_model(model, gpus=2) model.compile( loss='categorical_crossentropy', #loss='binary_crossentropy', #optimizer='adagrad', #optimizer=optimizers.RMSprop(lr=0.01), optimizer=optimizers.Adagrad(lr=0.01), metrics=['accuracy']) train_steps_per_epoch = math.ceil(train_generator.n / train_generator.batch_size) validation_steps = math.ceil(valid_generator.n / valid_generator.batch_size) print('train data size:', train_generator.n) print('train steps per epoch:', train_steps_per_epoch) print('valid data size:', valid_generator.n) print('validation_steps:', validation_steps) history = model.fit_generator(train_generator, steps_per_epoch=train_steps_per_epoch, epochs=300, validation_data=valid_generator, validation_steps=validation_steps)
])), ]) print('start pipeline') start_time = time.time() train = full_pipe.fit_transform(train) print("full_pipe fit_time: ", time.time() - start_time) start_time = time.time() model_in = Input(shape=(train.shape[1],), dtype='float32', sparse=True) out = layers.Dense(192, activation='relu')(model_in) out = layers.Dense(64, activation='relu')(out) out = layers.Dense(64, activation='relu')(out) out = layers.Dense(1)(out) model = Model(model_in, out) model.compile(optimizer=optimizers.Adagrad(), loss='mse') for i in range(3): model.fit(train, y, batch_size=2**(11+i), epochs=1, verbose=0) print("model fit_time: ",time.time()-start_time) del train gc.collect() print('start predicting') def load_test(): for df in pd.read_csv('../input/test_stg2.tsv', sep='\t', chunksize=700000): yield df
def main(): parser = argparse.ArgumentParser( description='simple 3D convolution for action recognition') parser.add_argument('--batch', type=int, default=32) parser.add_argument('--epoch', type=int, default=100) parser.add_argument('--videos', type=str, default='UCF101', help='directory where videos are stored') parser.add_argument('--nclass', type=int, default=249) parser.add_argument('--output', type=str, required=True) parser.add_argument('--skip', type=bool, default=True) parser.add_argument('--depth', type=int, default=16) args = parser.parse_args() #Initializing the dimentions of the frames img_rows, img_cols, frames = 32, 32, args.depth channel = 3 nb_classes = args.nclass fname_npz = 'dataset_test_{}_{}_{}.npz'.format(args.nclass, args.depth, args.skip) vid3d = videoto3d1.Videoto3D(img_rows, img_cols, frames) #If the dataset is already stored in npz file: if os.path.exists(fname_npz): loadeddata = np.load(fname_npz) X, Y = loadeddata["X"], loadeddata["Y"] else: #If not, we load the data with the helper function and save it for future use: x, y = loaddata(args.videos, vid3d, args.nclass, args.output, args.skip) Y = np_utils.to_categorical(y, nb_classes) X = x.reshape((x.shape[0], img_rows, img_cols, frames, channel)) X = X.astype('float32') np.savez(fname_npz, X=X, Y=Y) print('Saved test dataset to dataset_test.npz.') print('X_shape:{}\nY_shape:{}'.format(X.shape, Y.shape)) # Define model model = model_from_json( open('3dcnnresult/3dcnn_500_32_adam2.json', 'r').read()) model.load_weights('3dcnnresult/3dcnn_500_32_adam2.h5') model.summary() print("Loaded model from disk") #List of Optimizers we used: adam = optimizers.Adam(lr=0.01, decay=0.0001, amsgrad=False) sgd = optimizers.SGD(lr=0.001, momentum=0.9, decay=0.001, nesterov=True) ada = optimizers.Adagrad(lr=0.01, epsilon=None, decay=0.0) nadam = optimizers.Nadam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=None, schedule_decay=0.004) #Compiling and fitting the model model.compile(loss='categorical_crossentropy', optimizer=adam, metrics=['accuracy']) #Evaluating the model on the test_set loss, acc = model.evaluate(X, Y, verbose=1) print('Test loss:', loss) print('Test accuracy:', acc)
nb_classes = Y_train.shape[1] print(nb_classes, 'classes') rate = 0.1 model = Sequential() model.add(Dense(64, input_shape=(dims,), activation='relu')) model.add(Dropout(0.5, noise_shape=None, seed=42)) model.add(Dense(64, init='uniform', activation='relu')) model.add(Dropout(0.5, noise_shape=None, seed=42)) model.add(Dense(1, init='uniform', activation='sigmoid')) sgd = optimizers.SGD(lr=0.01, decay=0.005, momentum=0.5, nesterov=True) rmsprop = optimizers.RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.001) adagrad = optimizers.Adagrad(lr=0.01, epsilon=1e-09, decay=0.0001) adadelta = optimizers.Adadelta(lr=0.1, rho=0.95, epsilon=1e-08, decay=0.005) adamax = optimizers.Adamax(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.001) adam = optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.001) model.compile(optimizer=adagrad, loss='binary_crossentropy', metrics=["accuracy"]) model.summary() print("Model fitting...") fBestModel = 'best_model2.h5' #early_stop = EarlyStopping(monitor='val_loss', patience=500, verbose=1) best_model = ModelCheckpoint(fBestModel, verbose=0, save_best_only=True) model.fit(X_train, Y_train, validation_data = (X_val, Y_val), epochs=30000, batch_size=dims, verbose=True,
def get_optimizer(opt, decay=None, lr=None, momentum=0.0, nesterov=False, beta_1=0.9, beta_2=0.999, epsilon=1e-8, rho=None): """ get_optimizer is a wrapper for Keras optimizers. Parameters ---------- beta_1 : `float` adam optimizer parameter in range [0, 1) for updating bias first moment estimate beta_2 : `float` adam optimizer parameter in range [0, 1) for updating bias second moment estimate decay : `None` or `float` learning rate decay epsilon : `float` parameter for numerical stability opt : `str` Keras optimizer. Options: "sgd", "adam", "nadam", "rmsprop", "adagrad", "adamax" and "adadelta" lr : `None` or `float` optimizer learning rate momentum : `float` accelerate the gradient descent in the direction that dampens oscillations nesterov : `bool` use Nesterov Momentum rho : `None` or `float` gradient history Returns ------- optimizer : :class:`keras.optimizer` keras optimizer object """ ############################### # Stochastic Gradient Descent # ############################### if opt == 'sgd': if lr is None: lr = 0.01 if decay is None: decay = 0.0 optimizer = optimizers.SGD(lr=lr, momentum=momentum, decay=decay, nesterov=nesterov) ######## # Adam # ######## elif opt == 'adam': if lr is None: lr = 0.001 if decay is None: decay = 0.0 optimizer = optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay) ########## # Adamax # ########## elif opt == 'adamax': if lr is None: lr = 0.002 if decay is None: decay = 0.0 optimizer = optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay) ######### # Nadam # ######### # It is recommended to leave the parameters of this # optimizer at their default values. elif opt == 'nadam': if lr is None: lr = 0.002 if decay is None: decay = 0.004 optimizer = optimizers.Adam(lr=lr, beta_1=beta_1, beta_2=beta_2, epsilon=epsilon, decay=decay) ########### # RMSprop # ########### # It is recommended to leave the parameters of this # optimizer at their default values (except the learning # rate, which can be freely tuned). elif opt == 'rmsprop': if lr is None: lr = 0.001 if decay is None: decay = 0.0 if rho is None: rho = 0.9 optimizer = optimizers.RMSprop(lr=lr, rho=rho, epsilon=epsilon, decay=decay) ########### # Adagrad # ########### # It is recommended to leave the parameters of this # optimizer at their default values. elif opt == 'adagrad': if lr is None: lr = 0.01 if decay is None: decay = 0.0 optimizer = optimizers.Adagrad(lr=lr, decay=decay, epsilon=epsilon) ############ # Adadelta # ############ # It is recommended to leave the parameters of this # optimizer at their default values. elif opt == 'adadelta': if lr is None: lr = 1.0 if decay is None: decay = 0.0 if rho is None: rho = 0.95 optimizer = optimizers.Adadelta(lr=lr, rho=rho, epsilon=epsilon, decay=decay) else: print('ERROR: Unknown optimizer') sys.exit(1) return optimizer
def main(): global svrg; global adaGrad; # Set up data # -------- Make sure you change the featureSize variable below to the correct dimension -------- #result = loadData('a1a.t', 30956, 123) result = loadData('w8a.txt', 59245, 300) #result = loadData('covtype.libsvm.binary.scale', 581012, 54) examples = result[0] labels = result[1] #np.save("examples.npy", examples) #np.save("labels.npy", labels) #examples = np.load('examples.npy') #labels = np.load('labels.npy') global featureSize featureSize = 300 # Test Keras estimatedIters = testKeras(examples, labels) print("Choosing GD Plan...") while(1): print("0 - SGD") print("1 - Momentum") print("2 - Nesterov-Momentum") print("3 - Adagrad") print("4 - Adadelta") print("5 - RMSprop") print("6 - Adam") algoChoice = int(input("Choose Algo: ")) # Create Model for Keras model = Sequential() model.add(Dense(units=1, activation='linear', input_dim=featureSize)) # Choose GD Algorithm for Keras if (algoChoice == 0): myOpt = optimizers.SGD(lr=0.01, momentum=0., decay=0., nesterov=False) elif (algoChoice == 1): myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=False) elif (algoChoice == 2): myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=True) elif (algoChoice == 3): myOpt = optimizers.Adagrad(lr=0.01, epsilon=1e-6) elif (algoChoice == 4): myOpt = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-6) elif (algoChoice == 5): myOpt = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-6) elif (algoChoice == 6): myOpt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8) model.compile(optimizer=myOpt, loss=logloss) # To get results, use early stop, double iter. customCallback = PrintDeltaLossAndTime() myCallbacks = [customCallback] output = model.fit(examples, labels, epochs=estimatedIters[algoChoice], batch_size=int(len(examples)/50), callbacks=myCallbacks)
def train(): """ Def trains a Keras model of SqueezeDet and stores the checkpoint after each epoch """ #create config object cfg = load_dict(CONFIG) #create subdirs for logging of checkpoints and tensorboard stuff checkpoint_dir = log_dir_name + "/checkpoints" tb_dir = log_dir_name + "/tensorboard" #delete old checkpoints and tensorboard stuff if tf.gfile.Exists(checkpoint_dir) and cfg.init_file == 'none': tf.gfile.DeleteRecursively(checkpoint_dir) if tf.gfile.Exists(tb_dir): tf.gfile.DeleteRecursively(tb_dir) tf.gfile.MakeDirs(tb_dir) tf.gfile.MakeDirs(checkpoint_dir) #add stuff for documentation to config cfg.EPOCHS = EPOCHS cfg.OPTIMIZER = OPTIMIZER cfg.CUDA_VISIBLE_DEVICES = CUDA_VISIBLE_DEVICES cfg.GPUS = GPUS cfg.REDUCELRONPLATEAU = REDUCELRONPLATEAU if cfg.init_file != 'none': # ------------------ ImageTagger ----------------------- # img_file = 'imagetagger\\jpg\\train_jpg.txt' # gt_dir = 'imagetagger\\train_jpg.json' # base = "D:\\Humanoid\\squeezeDet\\Embedded_Object_Detection\\imagetagger\\jpg\\TRAIN\\" # ------------------ Small COCO ------------------------ img_file = 'dataset\\train_small.txt' gt_dir = 'dataset\\annotations\\train_small.json' base = "D:\\Humanoid\\squeezeDet\\Embedded_Object_Detection\\dataset\\train2017_small\\" else: # ---------------------- COCO ------------------------ # img_file = 'dataset\\images.txt' # gt_dir = 'dataset\\annotations\\ann_train_clean.json' # base = 'D:\\Humanoid\\squeezeDet\\Embedded_Object_Detection\\dataset\\train2017\\' # ------------------ Small COCO ------------------------ # img_file = 'dataset\\train_small.txt' # gt_dir = 'dataset\\annotations\\train_small.json' # base = "D:\\Humanoid\\squeezeDet\\Embedded_Object_Detection\\dataset\\train2017_small\\" # ------------------ ImageTagger ----------------------- img_file = 'imagetagger\\train_jpg.txt' gt_dir = 'imagetagger\\train_jpg.json' base = "D:\\Humanoid\\squeezeDet\\Embedded_Object_Detection\\imagetagger\\jpg\\TRAIN\\" #open files with images and ground truths files with full path names with open(img_file) as imgs: img_names = imgs.read().splitlines() imgs.close() #set gpu if GPUS < 2: os.environ['CUDA_VISIBLE_DEVICES'] = CUDA_VISIBLE_DEVICES else: gpus = "" for i in range(GPUS): gpus += str(i) + "," os.environ['CUDA_VISIBLE_DEVICES'] = gpus #scale batch size to gpus cfg.BATCH_SIZE = cfg.BATCH_SIZE * GPUS if STEPS is not None: nbatches_train = STEPS else: nbatches_train, _ = divmod(len(img_names), cfg.BATCH_SIZE) #tf config and session config = tf.ConfigProto(allow_soft_placement=True) sess = tf.Session(config=config) K.set_session(sess) #instantiate model squeeze = SqueezeDet(cfg) #callbacks cb = [] # print some run info print("Number of epochs: {}".format(EPOCHS)) print("Batch size: {}".format(cfg.BATCH_SIZE)) print("STEPS ", nbatches_train) # set optimizer # multiply by number of workers do adjust for increased batch size if OPTIMIZER == "adam": if cfg.init_file != 'none': cfg.LR = 1e-6 * GPUS opt = optimizers.Adam(lr=cfg.LR * GPUS, clipnorm=cfg.MAX_GRAD_NORM) print("Adam with learning rate ", cfg.LR) else: cfg.LR = 1e-5 * GPUS opt = optimizers.Adam(lr=cfg.LR * GPUS, clipnorm=cfg.MAX_GRAD_NORM) print("Adam with learning rate ", cfg.LR) elif OPTIMIZER == "rmsprop": opt = optimizers.RMSprop(lr=0.001 * GPUS, clipnorm=cfg.MAX_GRAD_NORM) cfg.LR = 0.001 * GPUS elif OPTIMIZER == "adagrad": opt = optimizers.Adagrad(lr=1.0 * GPUS, clipnorm=cfg.MAX_GRAD_NORM) cfg.LR = 1 * GPUS # use default if nothing was given else: # create sgd with momentum and gradient clipping cfg.LR = 1e-4 * GPUS opt = optimizers.SGD(lr=cfg.LR, decay=0, momentum=cfg.MOMENTUM, nesterov=False, clipnorm=cfg.MAX_GRAD_NORM) print("SGD with learning rate: {}".format(cfg.LR)) #add manuall learning rate decay #lrCallback = LearningRateScheduler(schedule) #cb.append(lrCallback) # save config file to log dir with open(log_dir_name + '/config.pkl', 'wb') as f: pickle.dump(cfg, f, pickle.HIGHEST_PROTOCOL) # add tensorboard callback tbCallBack = TensorBoard(log_dir=tb_dir, histogram_freq=0, write_graph=True, write_images=True) cb.append(tbCallBack) # if flag was given, add reducelronplateu callback # Reduce learning rate when a metric has stopped improving if REDUCELRONPLATEAU: reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.1, verbose=1, patience=5, min_lr=0.0) cb.append(reduce_lr) # print keras model summary if VERBOSE: print(squeeze.model.summary()) if cfg.init_file != "none": print("Weights initialized by name from {}".format(cfg.init_file)) load_only_possible_weights(squeeze.model, cfg.init_file, verbose=VERBOSE) # since these layers already existed in the ckpt they got loaded, you can reinitialized them. TODO set flag for that for layer in squeeze.model.layers: for v in layer.__dict__: v_arg = getattr(layer, v) if "fire10" in layer.name or "fire11" in layer.name or "conv12" in layer.name: if hasattr(v_arg, 'initializer'): initializer_method = getattr(v_arg, 'initializer') initializer_method.run(session=sess) print('reinitializing layer {}.{}'.format( layer.name, v)) #create train generator with open(gt_dir, 'r') as f: data = json.load(f) f.close() print("File read") train_generator = generator_from_data_path(img_names, data, base, config=cfg) #make model parallel if specified if GPUS > 1: #use multigpu model checkpoint ckp_saver = ModelCheckpointMultiGPU( checkpoint_dir + "/model.{epoch:02d}-{loss:.2f}.hdf5", monitor='loss', verbose=0, save_best_only=False, save_weights_only=True, mode='auto', period=1) cb.append(ckp_saver) print("Using multi gpu support with {} GPUs".format(GPUS)) # make the model parallel parallel_model = multi_gpu_model(squeeze.model, gpus=GPUS) parallel_model.compile(optimizer=opt, loss=[squeeze.loss], metrics=[ squeeze.loss_without_regularization, squeeze.bbox_loss, squeeze.class_loss, squeeze.conf_loss ]) #actually do the training parallel_model.fit_generator(train_generator, epochs=EPOCHS, steps_per_epoch=nbatches_train, callbacks=cb) else: # add a checkpoint saver ckp_saver = ModelCheckpoint(checkpoint_dir + "/model.{epoch:02d}-{loss:.2f}.hdf5", monitor='loss', verbose=0, save_best_only=False, save_weights_only=True, mode='auto', period=1) cb.append(ckp_saver) print("Using single GPU") #compile model from squeeze object, loss is not a function of model directly squeeze.model.compile(optimizer=opt, loss=[squeeze.loss], metrics=[ squeeze.loss_without_regularization, squeeze.bbox_loss, squeeze.class_loss, squeeze.conf_loss ]) #actually do the training squeeze.model.fit_generator(train_generator, epochs=EPOCHS, steps_per_epoch=nbatches_train, callbacks=cb, verbose=1) gc.collect()
def testKeras(examples, labels, subsetPercent = 0.2, desiredError = 0.001, timeLimit = 30): # Test each algorithm on a smaller dataset. exampleSubset = examples[0:int(len(examples)*subsetPercent)] labelSubset = labels[0:int(len(labels)*subsetPercent)] max_iterations = 10000 estimatedIters = [] allResults = [] for i in range(7): plt.figure(i+1) # Create Model for Keras model = Sequential() model.add(Dense(units=1, activation='linear', input_dim=featureSize)) # Choose GD Algorithm for Keras if (i == 0): myOpt = optimizers.SGD(lr=0.01, momentum=0., decay=0., nesterov=False) plt.title("SGD") elif (i == 1): myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=False) plt.title("Momentum") elif (i == 2): myOpt = optimizers.SGD(lr=0.01, momentum=0.9, decay=0., nesterov=True) plt.title("Nesterov-Momentum") elif (i == 3): myOpt = optimizers.Adagrad(lr=0.01, epsilon=1e-6) plt.title("Adagrad") elif (i == 4): myOpt = optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-6) plt.title("Adadelta") elif (i == 5): myOpt = optimizers.RMSprop(lr=0.001, rho=0.9, epsilon=1e-6) plt.title("RMSprop") elif (i == 6): myOpt = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-8) plt.title("Adam") model.compile(optimizer=myOpt, loss=logloss) # Create Custom Callback. Run GD. History saved in output. Use it to find the changes in loss per iteration customCallback = EarlyStoppingByDeltaLossOrTime(desiredError, timeLimit) myCallbacks = [customCallback] output = model.fit(exampleSubset, labelSubset, epochs=max_iterations, batch_size=int(len(exampleSubset)/50), callbacks=myCallbacks) losses = np.array(output.history['loss']) deltaLosses = -np.diff(losses) # Run again on the full dataset, for a few iterations. Use this to find the average time per iteration. # Reset callback to reset time elapsed and history of losses. model = Sequential() model.add(Dense(units=1, activation='linear', input_dim=featureSize)) model.compile(optimizer=myOpt, loss=logloss) customCallback = EarlyStoppingByDeltaLossOrTime(desiredError, timeLimit) myCallbacks = [customCallback] output = model.fit(examples, labels, epochs=5, batch_size=int(len(examples)/50), callbacks=myCallbacks) losses = np.array(output.history['loss']) timePerIter = myCallbacks[0].timeElapsed/len(losses) # Pass in the following: # 1. Array of DeltaLosses, iterations is length of array. # 2. Average Time per Iteration on the full dataset. results = fitGD(deltaLosses, timePerIter, desiredError) estimatedIters.append(results[0]) print("ETI: ", results[0]) print("ETA: ", results[1]) allResults.append(results) for i in range(len(allResults)): print("Algo", i, "Iterations:", allResults[i][0], "ETA:", allResults[i][1]) plt.show() return estimatedIters
feat_bilstm_c = Subtract()([bilstm_q1_c, bilstm_q2_c]) feat_bilstm_c = Lambda(lambda x: x**2)(feat_bilstm_c) # ------ last hidden ------- pair = Concatenate()([feat_bilstm_w, feat_bilstm_c]) pair = layer_dense1_c(pair) pair = Dropout(0.5)(pair) # ------ predict ------- predict = Dense(1, activation='sigmoid')(pair) LEARNING_RATE = 0.005 # optimizer = optimizers.SGD(lr=LEARNING_RATE, decay=1e-6, momentum=0.9, nesterov=True) # optimizer = optimizers.Adam(lr=LEARNING_RATE, beta_1=0.9, beta_2=0.999, epsilon=None, decay=1e-6, amsgrad=False) optimizer = optimizers.Adagrad(lr=LEARNING_RATE, epsilon=1e-06) model = Model(inputs=[input_q1_w, input_q1_c, input_q2_w, input_q2_c], outputs=predict) # model.compile(loss='binary_crossentropy', # optimizer=optimizer, # metrics=['binary_crossentropy']) model.compile(loss=focal_loss, optimizer=optimizer, metrics=['binary_crossentropy']) ############################################### # train ############################################### # train.test split import random random.shuffle(trainpair_idx)
def main(): args = read_args() X_train, X_test, y_train, y_test_orginal = load_dataset() # TODO 2: Convert the labels to categorical y_test_orginal_categ = keras.utils.to_categorical(y_test_orginal, 2) y_train_categ = keras.utils.to_categorical(y_train, 2) print() # TODO 3: Build the Keras model model = Sequential() # Add all the layers units_output_layer = y_test_orginal_categ.shape[1] #one per class if len(args.h_units) > 0: # Input to hidden layer model.add( Dense(args.h_units[0], input_dim=X_train.shape[1], kernel_regularizer=regularizers.l2(args.reg_l2))) model.add(Dropout(args.dropout[0])) model.add(Activation('relu')) for i in range(1, len(args.h_units)): model.add( Dense(args.h_units[i], input_dim=X_train.shape[1], kernel_regularizer=regularizers.l2(args.reg_l2))) model.add(Dropout(args.dropout[i])) model.add(Activation('relu')) # Hidden to output layer model.add(Dense(units_output_layer)) model.add(Activation('softmax')) else: dropout = 0 if len(args.dropout) > 0: dropout = args.dropout[0] model = Sequential([(Dense(units_output_layer, input_shape=(X_train.shape[1], ))), Activation('softmax')]) print(model.summary()) model.compile(loss='categorical_crossentropy', optimizer=optimizers.Adagrad(), metrics=['accuracy']) # TODO 4: Fit the model history = model.fit(X_train, y_train_categ, batch_size=args.batch_size, epochs=args.epochs) # TODO 5: Evaluate the model, calculating the metrics. # Option 1: Use the model.evaluate() method. For this, the model must be # already compiled with the metrics. #performance = model.evaluate(X_test) # Option 2: Use the model.predict() method and calculate the metrics using # sklearn. We recommend this, because you can store the predictions if # you need more analysis later. Also, if you calculate the metrics on a # notebook, then you can compare multiple classifiers. predictions = model.predict(X_test) for sample in predictions: if sample[0] < 0.5: sample[0] = 0 else: sample[0] = 1 if sample[1] < 0.5: sample[1] = 0 else: sample[1] = 1 acc = accuracy_score(predictions, y_test_orginal_categ) print('accuracy: ' + str(acc)) predictions = numpy.argmax(predictions, axis=None, out=None) # TODO 6: Save the results. results = pandas.DataFrame(y_test_orginal, columns=['y_test_orginal']) results.loc[:, 'predicted'] = predictions if args.experiment_name is None or args.experiment_name == "": args.experiment_name = str(args.batch_size) + "-" + str( args.epochs) + "-" + str(args.h_units) + "-" + str( args.reg_l2) + "-" + str(args.dropout) results.to_csv('predictions_{}.csv'.format(args.experiment_name), index=False) score_file = open('score_{}.txt'.format(args.experiment_name), 'w') score_file.write(str(acc) + '\n') score_file.close()
if algorithm == 'rmsprop': optimizer = opt.RMSprop(lr=0.001, rho=0.9, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif algorithm == 'sgd': optimizer = opt.SGD(lr=0.01, momentum=0.0, decay=0.0, nesterov=False, clipnorm=clipnorm, clipvalue=clipvalue) elif algorithm == 'adagrad': optimizer = opt.Adagrad(lr=0.01, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif algorithm == 'adadelta': optimizer = opt.Adadelta(lr=1.0, rho=0.95, epsilon=1e-06, clipnorm=clipnorm, clipvalue=clipvalue) elif algorithm == 'adam': optimizer = opt.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=clipnorm, clipvalue=clipvalue) elif algorithm == 'adamax':
BS = 10 LR = 0.001 DECAY = 0.00 EPSILON = 0.99 DROPOUT = 0.5 DROP = 0.90 EPOCHS_DROP = 25 NUM_TRAIN_IMAGES = 0 NUM_TEST_IMAGES = 0 # instantiate L1, L2 regularizers reg1 = regularizers.l1(0.02) reg2 = regularizers.l2(0.125) # optimizer parameters adagrad = optimizers.Adagrad(lr=LR, epsilon=EPSILON, decay=DECAY) # define data input generator: def csv_image_generator(inputPath, bs, lb, mode="train"): # open the CSV and PGS files for reading f = open(inputPath, "r") # loop indefinitely while True: # initialize our batches of images and labels images = [] labels = [] # keep looping until we reach our batch size while len(images) < bs: # images and PGSs have the same length # attempt to read the next line of the CSV file line = f.readline()
def test_adagrad(): _test_optimizer(optimizers.Adagrad()) _test_optimizer(optimizers.Adagrad(decay=1e-3))
train_col_stds = train_col_stds + ( train_col_stds == 0) * 1 # adds 1 where the stdev is 0 to not break division train_x = (train_x - train_col_means) / train_col_stds test_x = mens_data_sql_processed_with_history[features][ mens_data_sql_processed_with_history['date'] >= train_to_date].values test_x = test_x.astype(float) test_y = mens_data_sql_processed_with_history['player1Wins'][ mens_data_sql_processed_with_history['date'] >= train_to_date].values test_x = (test_x - train_col_means) / train_col_stds # optimizers optim_rmsprop = optimizers.RMSprop(lr=0.0001, rho=0.9) optim_sgd = optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.5, nesterov=True) optim_adagrad = optimizers.Adagrad(lr=0.01, decay=0) optim_adam = optimizers.Adam(lr=0.01, beta_1=0.9, beta_2=0.999, decay=0) # model number_epochs = 20 batch_sizes = 2**7 val_split = 0.1 dropout = 0.0 #weights = np.zeros(train_x.shape[0])+1 #weights = 0.9 + np.asarray(mens_data_sql_processed_with_history['player1Wins'][mens_data_sql_processed_with_history['date']<train_to_date]*0.2) weights = np.asarray(mens_data_sql_processed_with_history['b365P1'][ mens_data_sql_processed_with_history['date'] < train_to_date]) #weights = np.asarray(mens_data_sql_processed_with_history['b365P2'][mens_data_sql_processed_with_history['date']<train_to_date]) input_dimension = len(features)
def lstm_dense_sunspots(args): """ Main function """ # %% # IMPORTS # code repository sub-package imports from artificial_neural_networks.code.utils.download_monthly_sunspots import \ download_monthly_sunspots from artificial_neural_networks.code.utils.generic_utils import save_regress_model, \ series_to_supervised, affine_transformation from artificial_neural_networks.code.utils.vis_utils import regression_figs # %% if args.verbose > 0: print(args) # For reproducibility if args.reproducible: os.environ['PYTHONHASHSEED'] = '0' np.random.seed(args.seed) rn.seed(args.seed) tf.set_random_seed(args.seed) sess = tf.Session(graph=tf.get_default_graph()) K.set_session(sess) # print(hash("keras")) # %% # Load the Monthly sunspots dataset sunspots_path = download_monthly_sunspots() sunspots = np.genfromtxt(fname=sunspots_path, dtype=np.float32, delimiter=",", skip_header=1, usecols=1) # %% # Train-Test split L_series = len(sunspots) split_ratio = 2 / 3 # between zero and one n_split = int(L_series * split_ratio) look_back = args.look_back steps_ahead = args.steps_ahead train = sunspots[:n_split + (steps_ahead - 1)] test = sunspots[n_split - look_back:] train_x, train_y = series_to_supervised(train, look_back, steps_ahead) test_x, test_y = series_to_supervised(test, look_back, steps_ahead) train_y_series = train[look_back:train.shape[0] - (steps_ahead - 1)] test_y_series = test[look_back:] # %% # PREPROCESSING STEP scaling_factor = args.scaling_factor translation = args.translation n_train = train_x.shape[0] # number of training examples/samples n_test = test_x.shape[0] # number of test examples/samples n_in = train_x.shape[1] # number of features / dimensions n_out = train_y.shape[1] # number of steps ahead to be predicted # Reshape training and test sets train_x = train_x.reshape(n_train, n_in, 1) test_x = test_x.reshape(n_test, n_in, 1) # Apply preprocessing train_x_ = affine_transformation(train_x, scaling_factor, translation) train_y_ = affine_transformation(train_y, scaling_factor, translation) test_x_ = affine_transformation(test_x, scaling_factor, translation) test_y_ = affine_transformation(test_y, scaling_factor, translation) train_y_series_ = affine_transformation(train_y_series, scaling_factor, translation) test_y_series_ = affine_transformation(test_y_series, scaling_factor, translation) # %% # Model hyperparameters and ANN Architecture stateful = args.stateful if stateful: x = Input(shape=(n_in, 1), batch_shape=(1, n_in, 1)) # input layer else: x = Input(shape=(n_in, 1)) # input layer h = x h = LSTM(units=args.layer_size, stateful=stateful)(h) # hidden layer out = Dense(units=n_out, activation=None)(h) # output layer model = Model(inputs=x, outputs=out) if args.verbose > 0: model.summary() def root_mean_squared_error(y_true, y_pred): return K.sqrt(K.mean(K.square(y_pred - y_true), axis=-1)) loss_function = root_mean_squared_error metrics = ['mean_absolute_error', 'mean_absolute_percentage_error'] lr = args.lrearning_rate epsilon = args.epsilon optimizer_selection = { 'Adadelta': optimizers.Adadelta(lr=lr, rho=0.95, epsilon=epsilon, decay=0.0), 'Adagrad': optimizers.Adagrad(lr=lr, epsilon=epsilon, decay=0.0), 'Adam': optimizers.Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0, amsgrad=False), 'Adamax': optimizers.Adamax(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, decay=0.0), 'Nadam': optimizers.Nadam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=epsilon, schedule_decay=0.004), 'RMSprop': optimizers.RMSprop(lr=lr, rho=0.9, epsilon=epsilon, decay=0.0), 'SGD': optimizers.SGD(lr=lr, momentum=0.0, decay=0.0, nesterov=False) } optimizer = optimizer_selection[args.optimizer] model.compile(optimizer=optimizer, loss=loss_function, metrics=metrics) # %% # Save trained models for every epoch models_path = r'artificial_neural_networks/trained_models/' model_name = 'sunspots_lstm_dense' weights_path = models_path + model_name + '_weights' model_path = models_path + model_name + '_model' file_suffix = '_{epoch:04d}_{val_loss:.4f}_{val_mean_absolute_error:.4f}' if args.save_weights_only: file_path = weights_path else: file_path = model_path file_path += file_suffix monitor = 'val_loss' if args.save_models: checkpoint = ModelCheckpoint(file_path + '.h5', monitor=monitor, verbose=args.verbose, save_best_only=args.save_best, mode='auto', save_weights_only=args.save_weights_only) callbacks = [checkpoint] else: callbacks = [] # %% # TRAINING PHASE """ if stateful: shuffle = False else: shuffle = True """ if args.time_training: start = timer() for i in range(0, args.n_epochs): if args.verbose > 0: print('Epoch: {0}/{1}'.format(i + 1, args.n_epochs)) model.fit(x=train_x_, y=train_y_, validation_data=(test_x_, test_y_), batch_size=args.batch_size, epochs=1, verbose=args.verbose, callbacks=callbacks, shuffle=True) if stateful: model.reset_states() if args.time_training: end = timer() duration = end - start print('Total time for training (in seconds):') print(duration) # %% def model_predict(x_, y_): """ Predict using the LSTM Model (Multi-step ahead Forecasting) """ n_y_ = y_.shape[0] y_pred = np.zeros(n_y_) if args.recursive: # Recursive Strategy # TODO if args.verbose > 0: print('Following Recursive Strategy ...') n_x_ = x_.shape[0] n_iter = int(np.floor(n_x_ / steps_ahead)) L_last_window = n_x_ % steps_ahead first = 0 # Multi-step ahead Forecasting of all the full windows for i in range(0, n_iter): """ if args.verbose > 0: print('Completed: {0}/{1}'.format(i + 1, n_iter + 1)) """ pred_start = i * steps_ahead pred_end = pred_start + steps_ahead # first time step of each window (no recursion possible) j = pred_start k = j - pred_start # (always zero and unused) x_dyn = np.copy(x_[j:j + 1]) # use actual values only y_dyn = model.predict(x_dyn)[:, first] y_pred[j:j + 1] = y_dyn # remaining time steps of each window (with recursion) for j in range(pred_start + 1, pred_end): k = j - pred_start x_dyn = np.copy(x_[j:j + 1]) # use actual values (if possible) x_start = np.max([0, look_back - k]) y_start = np.max([0, k - look_back]) + pred_start # y_start = np.max([pred_start, j - look_back]) x_dyn[0, x_start:look_back, 0] = np.copy(y_pred[y_start:j]) # use pred. values y_dyn = model.predict(x_dyn)[:, first] # y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn() y_pred[j:j + 1] = np.max([0, y_dyn]) # y_pred[j:j + 1] = y_dyn # Multi-step ahead Forecasting of the last window if L_last_window > 0: """ if args.verbose > 0: print('Completed: {0}/{1}'.format(n_iter + 1, n_iter + 1)) """ pred_start = n_x_ - L_last_window pred_end = n_y_ # first time step of the last window (no recursion possible) j = pred_start k = j - pred_start # (always zero and unused) x_dyn = np.copy(x_[j:j + 1]) # use actual values only y_dyn = model.predict(x_dyn)[:, first] y_pred[j:j + 1] = y_dyn # remaining time steps of the last window (with recursion) for j in range(pred_start + 1, pred_end): k = j - pred_start x_dyn = np.roll(x_dyn, -1) # use act. values (if possible) x_start = np.max([0, look_back - k]) y_start = np.max([0, k - look_back]) + pred_start # y_start = np.max([pred_start, j - look_back]) x_dyn[0, x_start:look_back, 0] = np.copy(y_pred[y_start:j]) # use pred. values y_dyn = model.predict(x_dyn)[:, first] # y_after = np.max([0, y_dyn]) + 0.015 * np.random.randn() y_pred[j:j + 1] = np.max([0, y_dyn]) # y_pred[j:j + 1] = y_dyn """ # One-step ahead Forecasting n_x_ = x_.shape[0] for i in range(0, n_x_): x_dyn = x_[i:i+1] y_dyn = model.predict(x_dyn)[0, 0] y_pred[i] = y_dyn for i in range(n_x_, n_y): x_dyn[0, :, 0] = y_[i - look_back:i] y_dyn = model.predict(x_dyn)[0, 0] y_pred[i] = y_dyn """ else: # Multiple Ouptput Strategy # TODO if args.verbose > 0: print('Following Multiple Ouptput Strategy ...') n_iter = int(np.floor(n_y_ / steps_ahead)) L_last_window = n_y_ % steps_ahead y_dyn = x_[0, steps_ahead] # Multi-step ahead Forecasting of all the full windows for i in range(0, n_iter): pred_start = i * steps_ahead pred_end = pred_start + steps_ahead x_dyn = x_[pred_start:pred_start + 1] # TODO y_dyn = model.predict(x_dyn)[0] y_pred[pred_start:pred_end] = y_dyn # Multi-step ahead Forecasting of the last window if L_last_window > 0: pred_start = n_y_ - L_last_window pred_end = n_y_ x_dyn[0, :, 0] = y_[pred_end - look_back:pred_end] y_dyn = model.predict(x_dyn)[0] y_pred[pred_start:pred_end] = y_dyn[:L_last_window] return y_pred # %% # TESTING PHASE # Predict preprocessed values train_y_sum = [0] test_y_sum = [0] reps = 1 for i in range(reps): train_y_pred_ = model_predict(train_x_, train_y_series_) test_y_pred_ = model_predict(test_x_, test_y_series_) train_y_sum = np.sum([train_y_sum, train_y_pred_], axis=0) test_y_sum = np.sum([test_y_sum, test_y_pred_], axis=0) train_y_pred_ = train_y_sum / reps test_y_pred_ = test_y_sum / reps # Remove preprocessing train_y_pred = affine_transformation(train_y_pred_, scaling_factor, translation, inverse=True) test_y_pred = affine_transformation(test_y_pred_, scaling_factor, translation, inverse=True) train_rmse = sqrt(mean_squared_error(train_y_series, train_y_pred)) train_mae = mean_absolute_error(train_y_series, train_y_pred) train_r2 = r2_score(train_y_series, train_y_pred) test_rmse = sqrt(mean_squared_error(test_y_series, test_y_pred)) test_mae = mean_absolute_error(test_y_series, test_y_pred) test_r2 = r2_score(test_y_series, test_y_pred) if args.verbose > 0: print('Train RMSE: %.4f ' % (train_rmse)) print('Train MAE: %.4f ' % (train_mae)) print('Train (1 - R_squared): %.4f ' % (1.0 - train_r2)) print('Train R_squared: %.4f ' % (train_r2)) print('') print('Test RMSE: %.4f ' % (test_rmse)) print('Test MAE: %.4f ' % (test_mae)) print('Test (1 - R_squared): %.4f ' % (1.0 - test_r2)) print('Test R_squared: %.4f ' % (test_r2)) # %% # Data Visualization if args.plot: regression_figs(train_y=train_y_series, train_y_pred=train_y_pred, test_y=test_y_series, test_y_pred=test_y_pred) # %% # Save the architecture and the lastly trained model save_regress_model(model, models_path, model_name, weights_path, model_path, file_suffix, test_rmse, test_mae, args) # %% return model
def ANN_Model(X, Y, test_X, test_y): """----------测试集原数据作图----------------""" # plt.figure(0) # 创建图表1 # plt.title('observe') # plt.scatter([_ for _ in range(test_y.shape[0])], test_y) # 训练次数 # epochs = input('输入训练批次:\n') # loss_func = input('loss函数(' # 'mae[mean_absolute_error]\n' # 'mse[mean_squared_error]\n' # 'msle[mean_squared_logarithmic_error]\n' # 'squared_hinge[squared_hinge]\n' # 'logcosh[logcosh]\n' # '):\n') loss_func = 'mse' """----------配置网络模型----------------""" # 配置网络结构 model = Sequential() # hidden_units = input('隐藏层单元数量:\n') hidden_units = 20 # 第一隐藏层的配置:输入17,输出20 if layers_num == 1: model.add( Dense(hidden_units, input_dim=len(InputIndex), activation='sigmoid')) model.add(Dense(1, activation='sigmoid')) else: hidden_units1 = 20 hidden_units2 = 16 model.add( Dense(hidden_units1, input_dim=len(InputIndex), activation='sigmoid')) model.add(Dense(hidden_units2, activation='sigmoid')) model.add(Dense(1)) # 编译模型,指明代价函数和更新方法 Ada = optimizers.Adagrad(lr=0.018, epsilon=1e-06) model.compile(loss=loss_func, optimizer=Ada, metrics=[loss_func]) """----------训练模型--------------------""" print("training starts.....") model.fit(X, Y, epochs=epochs, verbose=1, batch_size=256) """----------评估模型--------------------""" # 用测试集去评估模型的准确度 cost = model.evaluate(test_X, test_y) print('\nTest accuracy:', cost) """----------模型存储--------------------""" save_model(model, weight_file_path) # 数据反归一化 trueTestYv = org_teY temp = model.predict(test_X).reshape(-1, 1) predTestYv = (temp.T * npscale.reshape(-1, 1)[-1, :] + npminthred.reshape(-1, 1)[-1, :]).T save_data = { 'Test': list(trueTestYv.T[0]), 'Predict': list(predTestYv.T[0]) } predict_predYv = pd.DataFrame(save_data) predict_predYv.to_csv('data/predict_test_value.csv') """----------计算R^2--------------------""" testYv = test_y.values.flatten() predYv = model.predict(test_X).flatten() slope, intercept, r_value, p_value, std_err = scipy.stats.linregress( testYv, predYv) print('R square is: ', r_value**2) # 数据反归一化 trueAllYv = org_target.values temp = model.predict(train).reshape(-1, 1) predAllYv = (temp.T * npscale.reshape(-1, 1)[-1, :] + npminthred.reshape(-1, 1)[-1, :]).T save_data = { 'TrueData': list(trueAllYv.T[0]), 'PredictData': list(predAllYv.T[0]) } predict_AllYv = pd.DataFrame(save_data) predict_AllYv.to_csv('data/predict_all_value.csv') # 求偏导,然后直接作图 # 获取每一层的权重 weights = {} for layer in model.layers: weight = layer.get_weights() info = layer.get_config() weights[info['name']] = weight if info['name'] == 'dense_1': df_weights = pd.DataFrame(weight[0].T, columns=InputIndex) else: df_weights = pd.DataFrame(weight[0].T) df_bias = pd.DataFrame(weight[1].T, columns=['bias']) df = pd.concat([df_weights, df_bias], axis=1) df.to_csv('weights/' + info['name'] + '.csv') res = [] for RawNo in range(train.shape[0]): TargetValue = list(train.loc[RawNo].values) x = UTPM.init_jacobian(TargetValue) # 求导函数,根据隐藏层数量选择 # 1层:DerivativeExpression_jac1 # 2层:DerivativeExpression_jac2 if layers_num == 1: y = DerivativeExpression_jac1(x, weights) else: y = DerivativeExpression_jac2(x, weights) algopy_jacobian = UTPM.extract_jacobian(y) # 最后一列插入NEE res.append(list(algopy_jacobian)) res = np.array(res) save_data = { 'TrueNEE': list(trueAllYv.T[0]), 'PredNEE': list(predAllYv.T[0]) } predict_AllYv = pd.DataFrame(save_data) deriColumns = ['d' + str(_) for _ in train.columns.tolist()] result = pd.DataFrame(res, columns=deriColumns) result = pd.concat([result, original_data, predict_AllYv], axis=1) result.to_csv('data/result_jacobian.csv') result.dropna(inplace=True) for i in range(len(InputIndex)): plt.figure(i) # 创建图表1 IndexName = InputIndex[i] # result = result[(result['d'+IndexName] > -5000) & (result['d'+IndexName] < 5000)] y = abs(result['d' + IndexName].values * scale[IndexName]) / result.shape[0] x = result[IndexName].values plt.xlabel(IndexName) plt.ylabel("NEE-" + IndexName) plt.scatter(x, y, s=1) plt.savefig("res/" + IndexName + ".png") plt.show()