def evaluate_error_model(X_tr_loc, Y_tr_loc, X_val, Y_val, params,pre_sample): start_time= time.time() batchsize = int(np.exp(float(params[2]))) #==> batch size model = build_model(params) nb_epochs = 2 ImportanceTraining(model, presample=pre_sample).fit(X_tr_loc, Y_tr_loc,batch_size=batchsize,epochs=nb_epochs,verbose=1, validation_data=(X_val, Y_val)) loss, score = model.evaluate(X_val, Y_val, batch_size=batchsize, verbose=1) print('Val error:', 1.0 - score) global obj_track obj_track.append( 1- score) global loss_track loss_track.append(loss) print("Obj Track: ",obj_track) print("Loss Track: ",loss_track) global iter_num iter_num = iter_num+1 print("Iter num:",iter_num) global best_obj best_obj = min(best_obj, 1- score) global best_loss best_loss = min(best_loss, loss) print("Best Error: ",best_obj) print("Best Loss:",best_loss) print("#######################") end_time = time.time()- start_time print("Time to run this hp:",end_time) print("#######################") return 1.0 - score
def test_checkpoint(self): m = Sequential( [Dense(10, activation="relu", input_shape=(2, )), Dense(2)]) m.compile("sgd", "mse") x = np.random.rand(32, 2) y = np.random.rand(32, 2) print(m.loss) ImportanceTraining(m).fit( x, y, epochs=1, callbacks=[ModelCheckpoint(path.join(self.tmpdir, "model.h5"))])
def test_simple_seq2seq(self): model = Sequential([ Embedding(100, 32, mask_zero=True, input_length=10), LSTM(32, return_sequences=True), LSTM(10, return_sequences=True), Activation("softmax") ]) model.compile("adam", "categorical_crossentropy") x = (np.random.rand(10, 10) * 100).astype(np.int32) y = np.random.rand(10, 10, 10) y /= y.sum(axis=-1, keepdims=True) ImportanceTraining(model).fit(x, y, batch_size=10)
def test_regularizers(self): reg = lambda w: 10 model = Sequential([ Dense(10, activation="relu", kernel_regularizer=reg, input_shape=(2, )), Dense(10, activation="relu", kernel_regularizer=reg), Dense(2) ]) model.compile("sgd", "mse") model = ImportanceTraining(model) history = model.fit(np.random.rand(64, 2), np.random.rand(64, 2)) self.assertGreater(history.history["loss"][0], 20.)
def test_simple_cifar(self): base = Xception(input_shape=(71, 71, 3), include_top=False, pooling="avg") y = Dense(10, activation="softmax")(base.output) model = Model(base.input, y) model.compile("sgd", "categorical_crossentropy", metrics=["accuracy"]) history = ImportanceTraining(model, presample=16).fit_generator( self._generate_images(batch_size=8), steps_per_epoch=10, epochs=1, batch_size=8) self.assertTrue("loss" in history.history) self.assertTrue("accuracy" in history.history) self.assertEqual(len(history.history["loss"]), 1) self.assertEqual(len(history.history["accuracy"]), 1)
def objective_function(params, s): nb_epochs = 50 start_time = time.time() depth = 28 width = 2 batch_size = 128 B_min = 2 B_max = 6 lr_initial_hp = float(np.exp(params[0])) decay_rate_factor_hp = float(np.exp(params[1])) l2_regular_weight_hp = float(np.exp(params[2])) momentum_hp = float(params[3]) # subseting the training data for Fabolas s_max = y_train.shape[0] shuffle = np.random.permutation(np.arange(s_max)) train_subset = x_train[shuffle[:s]] train_targets_subset = y_train[shuffle[:s]] model = wide_resnet(depth, width, l2_regular_weight_hp)(dset.shape, dset.output_size) model.compile(loss="categorical_crossentropy", optimizer=SGD(lr=lr_initial_hp, momentum=momentum_hp), metrics=["accuracy"]) # Create the data augmentation generator datagen = ImageDataGenerator( # set input mean to 0 over the dataset featurewise_center=False, # set each sample mean to 0 samplewise_center=False, # divide inputs by std of dataset featurewise_std_normalization=False, # divide each input by its std samplewise_std_normalization=False, # apply ZCA whitening zca_whitening=False, # randomly rotate images in the range (deg 0 to 180) rotation_range=0, # randomly shift images horizontally width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False) datagen.fit(train_subset) schedule = StepDecay(initAlpha=lr_initial_hp, factor=decay_rate_factor_hp, dropEvery=40) callbacks = [LearningRateScheduler(schedule)] global initial if initial < n_init_num: B = B_max else: B = np.random.uniform(low=B_min, high=B_max) initial = initial + 1 # Train the model ImportanceTraining(model, presample=float(B)).fit_generator( datagen.flow(train_subset, train_targets_subset, batch_size=batch_size), validation_data=(x_test, y_test), epochs=nb_epochs, verbose=1, batch_size=batch_size, callbacks=callbacks, steps_per_epoch=int(np.ceil(float(len(train_subset)) / batch_size))) loss, score = model.evaluate(x_test, y_test, verbose=1) c = time.time() - start_time print('Loss:', loss) print('Test error:', 1.0 - score) global obj_track obj_track.append(1 - score) global loss_track loss_track.append(loss) print("Obj Track: ", obj_track) print("Loss Track: ", loss_track) global iter_num iter_num = iter_num + 1 print("Iter num:", iter_num) global best_obj best_obj = min(best_obj, 1 - score) global best_loss best_loss = min(best_loss, loss) print("Best Error: ", best_obj) print("Best Loss:", best_loss) print("#######################") end_time = time.time() - start_time print("Time to run this hp:", end_time) print("#######################") return 1.0 - score, c
if not int(config['model']['use_saved_model']): # train new model import cnn, model_tools from callback import EarlyStoppingRankingAccuracy evaluation_function = EarlyStoppingRankingAccuracy(config, val_data) cnn.print_input(training_data) model = cnn.build_model(config, training_data, vocabulary, pretrained) if int(config['settings']['imp_tr']): #importance sampling from importance_sampling.training import ImportanceTraining logger.warning('Using truncated data!') fake_data_x = [a[:1000000] for a in training_data.x] hist = ImportanceTraining(model).fit(fake_data_x, training_data.y[:1000000], epochs=int( config['training']['epoch']), batch_size=100, callbacks=[evaluation_function]) else: logger.warning('Using truncated data!') fake_data_x = [a[:10000] for a in training_data.x] hist = model.fit(fake_data_x, training_data.y[:10000], epochs=int(config['training']['epoch']), batch_size=100, callbacks=[evaluation_function]) #hist = model.fit(training_data.x, training_data.y, epochs=int(config['training']['epoch']), batch_size=100, callbacks=[evaluation_function]) # WARNING (theano.tensor.blas): We did not found a dynamic library into the library_dir of the library we use for blas. If you use ATLAS, make sure to compile it with dynamics library. logger.info('Saving newly trained model...') model_tools.save_model(model, config['model']['path_model_architecture'],
model.add(Dense(num_classes, activation='softmax')) sgd = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) loss = keras.losses.categorical_crossentropy model.compile(loss=loss, optimizer=sgd, metrics=['accuracy']) def predict(sample, model): # sample = mfcc_operation(wavfile_path = filepath,max_len = feature_dim_2) sample_reshaped = sample.reshape(1, dim_1, dim_2, dim_3) return get_labels(big_numpy_files_path)[np.argmax( model.predict(sample_reshaped))] earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=15, verbose=verbose, mode='auto') ImportanceTraining(model).fit(X_train, y_train_hot, batch_size=batch_size, epochs=epochs, verbose=verbose, validation_data=(X_valid, y_valid_hot)) #model.fit(X_train, y_train_hot,batch_size=batch_size,callbacks=[earlystopping],epochs=epochs, verbose=verbose, validation_data=(X_valid, y_valid_hot)) #model.save('firstbigmodel.h5')
model = Sequential() model.add( Dense(512, activation='relu', kernel_regularizer=l2(1e-5), input_shape=(784, ))) model.add(Dense(512, activation='relu', kernel_regularizer=l2(1e-5))) model.add(Dense(10, kernel_regularizer=l2(1e-5))) model.add(Activation('softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) if args.importance_training: wrapped = ImportanceTraining(model, presample=5) else: wrapped = model history = wrapped.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1])
width_shift_range=0.1, # randomly shift images vertically height_shift_range=0.1, # randomly flip images horizontal_flip=True, # randomly flip images vertical_flip=False) datagen.fit(x_train) # Train the model if args.importance_training: ImportanceTraining(model).fit_generator( datagen.flow(x_train, y_train, batch_size=args.batch_size), validation_data=(x_test, y_test), epochs=10**6, verbose=1, callbacks=[training_schedule], batch_size=args.batch_size, steps_per_epoch=int(np.ceil(float(len(x_train)) / args.batch_size)) ) else: model.fit_generator( datagen.flow(x_train, y_train, batch_size=args.batch_size), validation_data=(x_test, y_test), epochs=10**6, verbose=1, callbacks=[training_schedule] ) # Score trained model. scores = model.evaluate(x_test, y_test, verbose=1)
print('Pad sequences (samples x time)') x_train = sequence.pad_sequences(x_train, maxlen=maxlen) x_test = sequence.pad_sequences(x_test, maxlen=maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) print('Build model...') model = Sequential() # we start off with an efficient embedding layer which maps # our vocab indices into embedding_dims dimensions model.add(Embedding(max_features, embedding_dims, input_length=maxlen)) # we add a GlobalAveragePooling1D, which will average the embeddings # of all words in the document model.add(GlobalAveragePooling1D()) # We project onto a single unit output layer, and squash it with a sigmoid: model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) ImportanceTraining(model).fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test))
def objective_function(param, s): start_time = time.time() nb_epochs = 50 B_min = 2 B_max = 6 # transform the input to the real range of the hyper-parameters, to be used for model training batch_size = int(param[0]) learning_rate = param[1] learning_rate_decay = param[2] l2_regular = param[3] conv_filters = int(param[4]) dense_units = int(param[5]) print("[parameters: batch_size: {0}/lr: {1}/lr_decay: {2}/l2: {3}/conv_filters: {4}/dense_unit: {5}]".format(\ batch_size, learning_rate, learning_rate_decay, l2_regular, conv_filters, dense_units)) num_conv_layers = 3 dropout_rate = 0.0 kernel_size = 5 pool_size = 3 # build the CNN model using Keras model = Sequential() model.add( Conv2D(conv_filters, (kernel_size, kernel_size), padding='same', input_shape=x_train.shape[1:], kernel_regularizer=regularizers.l2(l2_regular))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(Dropout(dropout_rate)) model.add( Conv2D(conv_filters, (kernel_size, kernel_size), padding='same', kernel_regularizer=regularizers.l2(l2_regular))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(Dropout(dropout_rate)) if num_conv_layers >= 3: model.add( Conv2D(conv_filters, (kernel_size, kernel_size), padding='same', kernel_regularizer=regularizers.l2(l2_regular))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(pool_size, pool_size))) model.add(Dropout(dropout_rate)) model.add(Flatten()) model.add( Dense(dense_units, kernel_regularizer=regularizers.l2(l2_regular))) model.add(Activation('relu')) model.add(Dropout(dropout_rate)) model.add(Dense(num_classes)) model.add(Activation('softmax')) opt = keras.optimizers.rmsprop(lr=learning_rate, decay=learning_rate_decay) model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) s = int(s) # s is not transformed in [0 1] # Shuffle the data and split up the request subset of the training data s_max = y_train.shape[0] shuffle = np.random.permutation(np.arange(s_max)) train_subset = x_train[shuffle[:s]] train_targets_subset = y_train[shuffle[:s]] global initial if initial < n_init_num: B = B_max else: B = np.random.uniform(low=B_min, high=B_max) initial = initial + 1 ImportanceTraining(model, presample=float(B)).fit(train_subset, train_targets_subset, batch_size=batch_size, epochs=nb_epochs, validation_data=(x_test, y_test), verbose=1) loss, score = model.evaluate(x_test, y_test, verbose=1) c = time.time() - start_time print('Loss:', loss) print('Val error:', 1.0 - score) global obj_track obj_track.append(1 - score) global loss_track loss_track.append(loss) print("Obj Track: ", obj_track) print("Loss Track: ", loss_track) global iter_num iter_num = iter_num + 1 print("Iter num:", iter_num) global best_obj best_obj = min(best_obj, 1 - score) global best_loss best_loss = min(best_loss, loss) print("Best Error: ", best_obj) print("Best Loss:", best_loss) print("#######################") end_time = time.time() - start_time print("Time to run this hp:", end_time) print("#######################") return 1.0 - score, c
# initiate RMSprop optimizer opt = keras.optimizers.Adam(lr=0.001) # Let's train the model using RMSprop model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) x_train = x_train.astype('float32') x_test = x_test.astype('float32') x_train /= 255 x_test /= 255 wrapped_model = ImportanceTraining(model, k=0.5, presample=64, adaptive_smoothing=True, smooth=0.5, forward_batch_size=64) if not data_augmentation: print('Not using data augmentation.') wrapped_model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True) else: print('Using real-time data augmentation.') # This will do preprocessing and realtime data augmentation: datagen = ImageDataGenerator( featurewise_center=False, # set input mean to 0 over the dataset
print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') # convert class vectors to binary class matrices y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) model = Sequential() model.add(Dense(512, activation='relu', input_shape=(784, ))) model.add(Dropout(0.2)) model.add(Dense(512, activation='relu')) model.add(Dropout(0.2)) model.add(Dense(10, activation='softmax')) model.summary() model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) history = ImportanceTraining(model, forward_batch_size=1024).fit( x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1])
if __name__ == "__main__": # Load the data dataset = CanevetICML2016(N=1024) x_train, y_train = dataset.train_data[:] x_test, y_test = dataset.test_data[:] y_train, y_test = y_train.argmax(axis=1), y_test.argmax(axis=1) # Create the NN and keep the initial weights model = create_nn() weights = model.get_weights() # Train with uniform sampling K.set_value(model.optimizer.lr, 0.01) model.fit(x_train, y_train, batch_size=64, epochs=10, validation_data=(x_test, y_test)) # Train with biased importance sampling model.set_weights(weights) K.set_value(model.optimizer.lr, 0.01) ImportanceTraining(model, forward_batch_size=1024).fit(x_train, y_train, batch_size=64, epochs=3, validation_data=(x_test, y_test))
print('x_test shape:', x_test.shape) print('Convert class vector to binary class matrix ' '(for use with categorical_crossentropy)') y_train = keras.utils.to_categorical(y_train, num_classes) y_test = keras.utils.to_categorical(y_test, num_classes) print('y_train shape:', y_train.shape) print('y_test shape:', y_test.shape) print('Building model...') model = Sequential() model.add(Dense(512, input_shape=(max_words, ))) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(num_classes)) model.add(Activation('softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) history = ImportanceTraining(model, k=1.0).fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_split=0.1) score = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=1) print('Test score:', score[0]) print('Test accuracy:', score[1])