def evaluate_error_model(X_tr_loc, Y_tr_loc, X_val, Y_val, params,pre_sample):
start_time= time.time()
batchsize = int(np.exp(float(params[2]))) #==> batch size
model = build_model(params)
nb_epochs = 2
ImportanceTraining(model, presample=pre_sample).fit(X_tr_loc, Y_tr_loc,batch_size=batchsize,epochs=nb_epochs,verbose=1, validation_data=(X_val, Y_val))
loss, score = model.evaluate(X_val, Y_val, batch_size=batchsize, verbose=1)
print('Val error:', 1.0 - score)
global obj_track
obj_track.append( 1- score)
global loss_track
loss_track.append(loss)
print("Obj Track: ",obj_track)
print("Loss Track: ",loss_track)
global iter_num
iter_num = iter_num+1
print("Iter num:",iter_num)
global best_obj
best_obj = min(best_obj, 1- score)
global best_loss
best_loss = min(best_loss, loss)
print("Best Error: ",best_obj)
print("Best Loss:",best_loss)
print("#######################")
end_time = time.time()- start_time
print("Time to run this hp:",end_time)
print("#######################")
return 1.0 - score
def test_checkpoint(self):
m = Sequential(
[Dense(10, activation="relu", input_shape=(2, )),
Dense(2)])
m.compile("sgd", "mse")
x = np.random.rand(32, 2)
y = np.random.rand(32, 2)
print(m.loss)
ImportanceTraining(m).fit(
x,
y,
epochs=1,
callbacks=[ModelCheckpoint(path.join(self.tmpdir, "model.h5"))])
def test_simple_seq2seq(self):
model = Sequential([
Embedding(100, 32, mask_zero=True, input_length=10),
LSTM(32, return_sequences=True),
LSTM(10, return_sequences=True),
Activation("softmax")
])
model.compile("adam", "categorical_crossentropy")
x = (np.random.rand(10, 10) * 100).astype(np.int32)
y = np.random.rand(10, 10, 10)
y /= y.sum(axis=-1, keepdims=True)
ImportanceTraining(model).fit(x, y, batch_size=10)
def test_regularizers(self):
reg = lambda w: 10
model = Sequential([
Dense(10,
activation="relu",
kernel_regularizer=reg,
input_shape=(2, )),
Dense(10, activation="relu", kernel_regularizer=reg),
Dense(2)
])
model.compile("sgd", "mse")
model = ImportanceTraining(model)
history = model.fit(np.random.rand(64, 2), np.random.rand(64, 2))
self.assertGreater(history.history["loss"][0], 20.)
def test_simple_cifar(self):
base = Xception(input_shape=(71, 71, 3),
include_top=False,
pooling="avg")
y = Dense(10, activation="softmax")(base.output)
model = Model(base.input, y)
model.compile("sgd", "categorical_crossentropy", metrics=["accuracy"])
history = ImportanceTraining(model, presample=16).fit_generator(
self._generate_images(batch_size=8),
steps_per_epoch=10,
epochs=1,
batch_size=8)
self.assertTrue("loss" in history.history)
self.assertTrue("accuracy" in history.history)
self.assertEqual(len(history.history["loss"]), 1)
self.assertEqual(len(history.history["accuracy"]), 1)
def objective_function(params, s):
nb_epochs = 50
start_time = time.time()
depth = 28
width = 2
batch_size = 128
B_min = 2
B_max = 6
lr_initial_hp = float(np.exp(params[0]))
decay_rate_factor_hp = float(np.exp(params[1]))
l2_regular_weight_hp = float(np.exp(params[2]))
momentum_hp = float(params[3])
# subseting the training data for Fabolas
s_max = y_train.shape[0]
shuffle = np.random.permutation(np.arange(s_max))
train_subset = x_train[shuffle[:s]]
train_targets_subset = y_train[shuffle[:s]]
model = wide_resnet(depth, width, l2_regular_weight_hp)(dset.shape,
dset.output_size)
model.compile(loss="categorical_crossentropy",
optimizer=SGD(lr=lr_initial_hp, momentum=momentum_hp),
metrics=["accuracy"])
# Create the data augmentation generator
datagen = ImageDataGenerator(
# set input mean to 0 over the dataset
featurewise_center=False,
# set each sample mean to 0
samplewise_center=False,
# divide inputs by std of dataset
featurewise_std_normalization=False,
# divide each input by its std
samplewise_std_normalization=False,
# apply ZCA whitening
zca_whitening=False,
# randomly rotate images in the range (deg 0 to 180)
rotation_range=0,
# randomly shift images horizontally
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False)
datagen.fit(train_subset)
schedule = StepDecay(initAlpha=lr_initial_hp,
factor=decay_rate_factor_hp,
dropEvery=40)
callbacks = [LearningRateScheduler(schedule)]
global initial
if initial < n_init_num:
B = B_max
else:
B = np.random.uniform(low=B_min, high=B_max)
initial = initial + 1
# Train the model
ImportanceTraining(model, presample=float(B)).fit_generator(
datagen.flow(train_subset, train_targets_subset,
batch_size=batch_size),
validation_data=(x_test, y_test),
epochs=nb_epochs,
verbose=1,
batch_size=batch_size,
callbacks=callbacks,
steps_per_epoch=int(np.ceil(float(len(train_subset)) / batch_size)))
loss, score = model.evaluate(x_test, y_test, verbose=1)
c = time.time() - start_time
print('Loss:', loss)
print('Test error:', 1.0 - score)
global obj_track
obj_track.append(1 - score)
global loss_track
loss_track.append(loss)
print("Obj Track: ", obj_track)
print("Loss Track: ", loss_track)
global iter_num
iter_num = iter_num + 1
print("Iter num:", iter_num)
global best_obj
best_obj = min(best_obj, 1 - score)
global best_loss
best_loss = min(best_loss, loss)
print("Best Error: ", best_obj)
print("Best Loss:", best_loss)
print("#######################")
end_time = time.time() - start_time
print("Time to run this hp:", end_time)
print("#######################")
return 1.0 - score, c
if not int(config['model']['use_saved_model']): # train new model
import cnn, model_tools
from callback import EarlyStoppingRankingAccuracy
evaluation_function = EarlyStoppingRankingAccuracy(config, val_data)
cnn.print_input(training_data)
model = cnn.build_model(config, training_data, vocabulary, pretrained)
if int(config['settings']['imp_tr']):
#importance sampling
from importance_sampling.training import ImportanceTraining
logger.warning('Using truncated data!')
fake_data_x = [a[:1000000] for a in training_data.x]
hist = ImportanceTraining(model).fit(fake_data_x,
training_data.y[:1000000],
epochs=int(
config['training']['epoch']),
batch_size=100,
callbacks=[evaluation_function])
else:
logger.warning('Using truncated data!')
fake_data_x = [a[:10000] for a in training_data.x]
hist = model.fit(fake_data_x,
training_data.y[:10000],
epochs=int(config['training']['epoch']),
batch_size=100,
callbacks=[evaluation_function])
#hist = model.fit(training_data.x, training_data.y, epochs=int(config['training']['epoch']), batch_size=100, callbacks=[evaluation_function])
# WARNING (theano.tensor.blas): We did not found a dynamic library into the library_dir of the library we use for blas. If you use ATLAS, make sure to compile it with dynamics library.
logger.info('Saving newly trained model...')
model_tools.save_model(model, config['model']['path_model_architecture'],
model.add(Dense(num_classes, activation='softmax'))
sgd = keras.optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
loss = keras.losses.categorical_crossentropy
model.compile(loss=loss, optimizer=sgd, metrics=['accuracy'])
def predict(sample, model):
# sample = mfcc_operation(wavfile_path = filepath,max_len = feature_dim_2)
sample_reshaped = sample.reshape(1, dim_1, dim_2, dim_3)
return get_labels(big_numpy_files_path)[np.argmax(
model.predict(sample_reshaped))]
earlystopping = EarlyStopping(monitor='val_loss',
min_delta=0.001,
patience=15,
verbose=verbose,
mode='auto')
ImportanceTraining(model).fit(X_train,
y_train_hot,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
validation_data=(X_valid, y_valid_hot))
#model.fit(X_train, y_train_hot,batch_size=batch_size,callbacks=[earlystopping],epochs=epochs, verbose=verbose, validation_data=(X_valid, y_valid_hot))
#model.save('firstbigmodel.h5')
model = Sequential()
model.add(
Dense(512,
activation='relu',
kernel_regularizer=l2(1e-5),
input_shape=(784, )))
model.add(Dense(512, activation='relu', kernel_regularizer=l2(1e-5)))
model.add(Dense(10, kernel_regularizer=l2(1e-5)))
model.add(Activation('softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
if args.importance_training:
wrapped = ImportanceTraining(model, presample=5)
else:
wrapped = model
history = wrapped.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
width_shift_range=0.1,
# randomly shift images vertically
height_shift_range=0.1,
# randomly flip images
horizontal_flip=True,
# randomly flip images
vertical_flip=False)
datagen.fit(x_train)
# Train the model
if args.importance_training:
ImportanceTraining(model).fit_generator(
datagen.flow(x_train, y_train, batch_size=args.batch_size),
validation_data=(x_test, y_test),
epochs=10**6,
verbose=1,
callbacks=[training_schedule],
batch_size=args.batch_size,
steps_per_epoch=int(np.ceil(float(len(x_train)) / args.batch_size))
)
else:
model.fit_generator(
datagen.flow(x_train, y_train, batch_size=args.batch_size),
validation_data=(x_test, y_test),
epochs=10**6,
verbose=1,
callbacks=[training_schedule]
)
# Score trained model.
scores = model.evaluate(x_test, y_test, verbose=1)
print('Pad sequences (samples x time)')
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('Build model...')
model = Sequential()
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
# we add a GlobalAveragePooling1D, which will average the embeddings
# of all words in the document
model.add(GlobalAveragePooling1D())
# We project onto a single unit output layer, and squash it with a sigmoid:
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
ImportanceTraining(model).fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test))
def objective_function(param, s):
start_time = time.time()
nb_epochs = 50
B_min = 2
B_max = 6
# transform the input to the real range of the hyper-parameters, to be used for model training
batch_size = int(param[0])
learning_rate = param[1]
learning_rate_decay = param[2]
l2_regular = param[3]
conv_filters = int(param[4])
dense_units = int(param[5])
print("[parameters: batch_size: {0}/lr: {1}/lr_decay: {2}/l2: {3}/conv_filters: {4}/dense_unit: {5}]".format(\
batch_size, learning_rate, learning_rate_decay, l2_regular, conv_filters, dense_units))
num_conv_layers = 3
dropout_rate = 0.0
kernel_size = 5
pool_size = 3
# build the CNN model using Keras
model = Sequential()
model.add(
Conv2D(conv_filters, (kernel_size, kernel_size),
padding='same',
input_shape=x_train.shape[1:],
kernel_regularizer=regularizers.l2(l2_regular)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Dropout(dropout_rate))
model.add(
Conv2D(conv_filters, (kernel_size, kernel_size),
padding='same',
kernel_regularizer=regularizers.l2(l2_regular)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Dropout(dropout_rate))
if num_conv_layers >= 3:
model.add(
Conv2D(conv_filters, (kernel_size, kernel_size),
padding='same',
kernel_regularizer=regularizers.l2(l2_regular)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Dropout(dropout_rate))
model.add(Flatten())
model.add(
Dense(dense_units, kernel_regularizer=regularizers.l2(l2_regular)))
model.add(Activation('relu'))
model.add(Dropout(dropout_rate))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
opt = keras.optimizers.rmsprop(lr=learning_rate, decay=learning_rate_decay)
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
s = int(s) # s is not transformed in [0 1]
# Shuffle the data and split up the request subset of the training data
s_max = y_train.shape[0]
shuffle = np.random.permutation(np.arange(s_max))
train_subset = x_train[shuffle[:s]]
train_targets_subset = y_train[shuffle[:s]]
global initial
if initial < n_init_num:
B = B_max
else:
B = np.random.uniform(low=B_min, high=B_max)
initial = initial + 1
ImportanceTraining(model, presample=float(B)).fit(train_subset,
train_targets_subset,
batch_size=batch_size,
epochs=nb_epochs,
validation_data=(x_test,
y_test),
verbose=1)
loss, score = model.evaluate(x_test, y_test, verbose=1)
c = time.time() - start_time
print('Loss:', loss)
print('Val error:', 1.0 - score)
global obj_track
obj_track.append(1 - score)
global loss_track
loss_track.append(loss)
print("Obj Track: ", obj_track)
print("Loss Track: ", loss_track)
global iter_num
iter_num = iter_num + 1
print("Iter num:", iter_num)
global best_obj
best_obj = min(best_obj, 1 - score)
global best_loss
best_loss = min(best_loss, loss)
print("Best Error: ", best_obj)
print("Best Loss:", best_loss)
print("#######################")
end_time = time.time() - start_time
print("Time to run this hp:", end_time)
print("#######################")
return 1.0 - score, c
# initiate RMSprop optimizer
opt = keras.optimizers.Adam(lr=0.001)
# Let's train the model using RMSprop
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255
wrapped_model = ImportanceTraining(model,
k=0.5,
presample=64,
adaptive_smoothing=True,
smooth=0.5,
forward_batch_size=64)
if not data_augmentation:
print('Not using data augmentation.')
wrapped_model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(x_test, y_test),
shuffle=True)
else:
print('Using real-time data augmentation.')
# This will do preprocessing and realtime data augmentation:
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
model = Sequential()
model.add(Dense(512, activation='relu', input_shape=(784, )))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(10, activation='softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
history = ImportanceTraining(model, forward_batch_size=1024).fit(
x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
if __name__ == "__main__":
# Load the data
dataset = CanevetICML2016(N=1024)
x_train, y_train = dataset.train_data[:]
x_test, y_test = dataset.test_data[:]
y_train, y_test = y_train.argmax(axis=1), y_test.argmax(axis=1)
# Create the NN and keep the initial weights
model = create_nn()
weights = model.get_weights()
# Train with uniform sampling
K.set_value(model.optimizer.lr, 0.01)
model.fit(x_train,
y_train,
batch_size=64,
epochs=10,
validation_data=(x_test, y_test))
# Train with biased importance sampling
model.set_weights(weights)
K.set_value(model.optimizer.lr, 0.01)
ImportanceTraining(model,
forward_batch_size=1024).fit(x_train,
y_train,
batch_size=64,
epochs=3,
validation_data=(x_test,
y_test))
print('x_test shape:', x_test.shape)
print('Convert class vector to binary class matrix '
'(for use with categorical_crossentropy)')
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
print('y_train shape:', y_train.shape)
print('y_test shape:', y_test.shape)
print('Building model...')
model = Sequential()
model.add(Dense(512, input_shape=(max_words, )))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = ImportanceTraining(model, k=1.0).fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_split=0.1)
score = model.evaluate(x_test, y_test, batch_size=batch_size, verbose=1)
print('Test score:', score[0])
print('Test accuracy:', score[1])
