def test_image_data_generator_training():
np.random.seed(1337)
img_gen = ImageDataGenerator(rescale=1.) # Dummy ImageDataGenerator
input_shape = (16, 16, 3)
(x_train, y_train), (x_test, y_test) = get_test_data(num_train=500,
num_test=200,
input_shape=input_shape,
classification=True,
num_classes=4)
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
model = Sequential([
layers.Conv2D(filters=8, kernel_size=3,
activation='relu',
input_shape=input_shape),
layers.MaxPooling2D(pool_size=2),
layers.Conv2D(filters=4, kernel_size=(3, 3),
activation='relu', padding='same'),
layers.GlobalAveragePooling2D(),
layers.Dense(y_test.shape[-1], activation='softmax')
])
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
history = model.fit_generator(img_gen.flow(x_train, y_train, batch_size=16),
epochs=10,
validation_data=img_gen.flow(x_test, y_test,
batch_size=16),
verbose=0)
assert history.history['val_acc'][-1] > 0.75
model.evaluate_generator(img_gen.flow(x_train, y_train, batch_size=16))
def train_CAE():
encoder = containers.Sequential()
encoder.add(Permute((3,1,2),input_shape=(h,w,ch))) # reorder input to ch, h, w (no sample axis)
encoder.add(GaussianNoise(0.05)) # corrupt inputs slightly
encoder.add(Convolution2D(16,3,3,init='glorot_uniform',border_mode='same'))
encoder.add(MaxPooling2D((2,2)))
encoder.add(Activation('tanh'))
encoder.add(Convolution2D(32,3,3,init='glorot_uniform',border_mode='same'))
encoder.add(MaxPooling2D((2,2)))
encoder.add(Activation('tanh'))
decoder = containers.Sequential()
decoder.add(UpSampling2D((2,2),input_shape=(32,32,32)))
decoder.add(Convolution2D(3,3,3,init='glorot_uniform',border_mode='same'))
decoder.add(Activation('tanh'))
decoder.add(UpSampling2D((2,2),input_shape=(16,64,64)))
decoder.add(Convolution2D(3,3,3,init='glorot_uniform',border_mode='same'))
decoder.add(Activation('tanh'))
decoder.add(Permute((2,3,1)))
autoencoder = AutoEncoder(encoder,decoder)
model = Sequential()
model.add(autoencoder)
model.compile(optimizer='rmsprop', loss='mae')
# if shapes don't match, check the output_shape of encoder/decoder
genr = image_generator(biz_id_train['photo_id'], batch_size)
model.fit_generator(genr, samples_per_epoch=len(biz_id_train), nb_epoch=10)
def train(dataReader, oneHot, oneHotAveraged, contextHashes): n = (Epochs + 1) * SamplesPerEpoch # TODO + 1 should not be needed tokeniser = Tokenizer(nb_words=MaxWords) tokeniser.fit_on_texts((row[0] for row in dataReader.trainingData(n))) # `word_index` maps each word to its unique index dictionarySize = len(tokeniser.word_index) + 1 oneHotDimension = (1 if oneHotAveraged else SequenceLength) * dictionarySize if oneHot else 0 contextHashesDimension = dictionarySize * 2 if contextHashes else 0 model = Sequential() model.add(Dense(EmbeddingDim, input_dim=(oneHotDimension + contextHashesDimension))) model.add(Dense(Labels, activation='softmax')) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) trainingGenerator = mapGenerator(dataReader.trainingData(n), tokeniser, dictionarySize, oneHot, oneHotAveraged, contextHashes) validationGenerator = mapGenerator(dataReader.validationData(n), tokeniser, dictionarySize, oneHot, oneHotAveraged, contextHashes) model.fit_generator(trainingGenerator, nb_epoch=Epochs, samples_per_epoch=SamplesPerEpoch, validation_data=validationGenerator, nb_val_samples=SamplesPerEpoch) model2 = Sequential() model2.add(Dense(EmbeddingDim, input_dim=(oneHotDimension + contextHashesDimension), weights=model.layers[0].get_weights())) return model, model2, tokeniser, dictionarySize
def test_multiprocessing_fit_error():
batch_size = 10
good_batches = 3
def custom_generator():
"""Raises an exception after a few good batches"""
for i in range(good_batches):
yield (np.random.randint(batch_size, 256, (50, 2)),
np.random.randint(batch_size, 2, 50))
raise RuntimeError
model = Sequential()
model.add(Dense(1, input_shape=(2, )))
model.compile(loss='mse', optimizer='adadelta')
samples = batch_size * (good_batches + 1)
with pytest.raises(StopIteration):
model.fit_generator(
custom_generator(), samples, 1,
workers=4, use_multiprocessing=True,
)
with pytest.raises(StopIteration):
model.fit_generator(
custom_generator(), samples, 1,
use_multiprocessing=False,
)
def main():
ext = extension_from_parameters()
out_dim = 1
loss = 'mse'
metrics = None
#metrics = ['accuracy'] if CATEGORICAL else None
datagen = RegressionDataGenerator()
train_gen = datagen.flow(batch_size=BATCH_SIZE)
val_gen = datagen.flow(val=True, batch_size=BATCH_SIZE)
val_gen2 = datagen.flow(val=True, batch_size=BATCH_SIZE)
model = Sequential()
for layer in LAYERS:
if layer:
model.add(Dense(layer, input_dim=datagen.input_dim, activation=ACTIVATION))
if DROP:
model.add(Dropout(DROP))
model.add(Dense(out_dim))
model.summary()
model.compile(loss=loss, optimizer='rmsprop', metrics=metrics)
train_samples = int(datagen.n_train/BATCH_SIZE) * BATCH_SIZE
val_samples = int(datagen.n_val/BATCH_SIZE) * BATCH_SIZE
history = BestLossHistory(val_gen2, val_samples, ext)
checkpointer = ModelCheckpoint(filepath='model'+ext+'.h5', save_best_only=True)
model.fit_generator(train_gen, train_samples,
nb_epoch = NB_EPOCH,
validation_data = val_gen,
nb_val_samples = val_samples,
callbacks=[history, checkpointer])
class CNN(object):
def __init__(self):
self.model = Sequential([
Conv2D(50, (3, 3), input_shape=(28, 28, 1), padding='same', activation='relu'),
Conv2D(50, (3, 3), input_shape=(28, 28, 1), padding='same', activation='relu'),
MaxPool2D(pool_size=(4, 4), strides=(3, 3), padding='same'),
Conv2D(32, (3, 3), padding='same', activation='relu' ),
Conv2D(32, (3, 3), padding='same', activation='relu' ),
MaxPool2D(pool_size=(7, 7), strides=(3, 3), padding='same'),
Flatten(),
Dropout(0.5),
Dense(64, activation='relu'),
Dense(10, activation='softmax')
])
self.model.compile(loss=categorical_crossentropy, optimizer=Adam(), metrics=['accuracy'])
def train(self):
mnist = input_data.read_data_sets("../MNIST_data/", one_hot=True)
train_datagen = ImageDataGenerator(rotation_range=20, width_shift_range=0.2, height_shift_range=0.2)
train_datagen.fit(mnist.train.images.reshape(-1, 28, 28, 1))
x_test, y_test = mnist.test.images.reshape(-1, 28, 28, 1), mnist.test.labels
self.model.fit_generator(train_datagen.flow(mnist.train.images.reshape(-1, 28, 28, 1), mnist.train.labels),
#batch_size=128,
epochs=20,
verbose=1,
validation_data=(x_test, y_test),
callbacks=[TrainValTensorBoard(log_dir='./logs/cnn4', histogram_freq=1, write_grads=True)])
score = self.model.evaluate(x_test, y_test, verbose=0)
print('Loss', score[0], 'acc', score[1])
def test_multiprocessing_fit_error():
batch_size = 32
good_batches = 5
def myGenerator():
"""Raises an exception after a few good batches"""
for i in range(good_batches):
yield (np.random.randint(batch_size, 256, (500, 2)),
np.random.randint(batch_size, 2, 500))
raise RuntimeError
model = Sequential()
model.add(Dense(1, input_shape=(2, )))
model.compile(loss='mse', optimizer='adadelta')
samples = batch_size * (good_batches + 1)
with pytest.raises(Exception):
model.fit_generator(
myGenerator(), samples, 1,
nb_worker=4, pickle_safe=True,
)
with pytest.raises(Exception):
model.fit_generator(
myGenerator(), samples, 1,
pickle_safe=False,
)
class MLP(BaseEstimator):
def __init__(self, verbose=0, model=None, final_activation='sigmoid'):
self.verbose = verbose
self.model = model
self.final_activation = final_activation
def fit(self, X, y):
if not self.model:
self.model = Sequential()
self.model.add(Dense(1000, input_dim=X.shape[1]))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.5))
self.model.add(Dense(y.shape[1]))
self.model.add(Activation(self.final_activation))
self.model.compile(loss='categorical_crossentropy', optimizer=Adam(lr=0.01))
self.model.fit_generator(generator=_batch_generator(X, y, 256, True),
samples_per_epoch=X.shape[0], nb_epoch=20, verbose=self.verbose)
def predict(self, X):
pred = self.predict_proba(X)
return sparse.csr_matrix(pred > 0.2)
def predict_proba(self, X):
pred = self.model.predict_generator(generator=_batch_generatorp(X, 512), val_samples=X.shape[0])
return pred
def test_CallbackValData():
np.random.seed(1337)
(X_train, y_train), (X_test, y_test) = get_data_callbacks()
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
model = Sequential()
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
cbk = callbacks.LambdaCallback(on_train_end=lambda x: 1)
model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=[cbk], epochs=1)
cbk2 = callbacks.LambdaCallback(on_train_end=lambda x: 1)
train_generator = data_generator(X_train, y_train, batch_size)
model.fit_generator(train_generator, len(X_train), epochs=1,
validation_data=(X_test, y_test),
callbacks=[cbk2])
# callback validation data should always have x, y, and sample weights
assert len(cbk.validation_data) == len(cbk2.validation_data) == 3
assert cbk.validation_data[0] is cbk2.validation_data[0]
assert cbk.validation_data[1] is cbk2.validation_data[1]
assert cbk.validation_data[2].shape == cbk2.validation_data[2].shape
def CNN(trainDir, validationDir, classNum):
model = Sequential()
model.add(Convolution2D(4, 3, 3, input_shape=(img_width, img_height, 1)))
model.add(Activation('relu'))
model.add(Convolution2D(4, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# layer
model.add(Convolution2D(8, 3, 3))
model.add(Activation('relu'))
model.add(Convolution2D(8, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Convolution2D(16, 3, 3))
# model.add(Activation('relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# layer
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
# model.add(Dropout(0.5))
model.add(Dense(16))
model.add(Activation('relu'))
model.add(Dropout(0.6))
model.add(Dense(classNum))
model.add(Activation('softmax'))
# test
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
rescale=1./255,
shear_range=0.2,
zca_whitening=True,
zoom_range=0.2,
horizontal_flip=False)
# this is the augmentation configuration we will use for testing:
# only rescaling
test_datagen = ImageDataGenerator(rescale=1./255, zca_whitening=True)
train_generator = train_datagen.flow_from_directory(
trainDir,
target_size=(img_width, img_height),
batch_size=32,
color_mode='grayscale',
class_mode='categorical')
validation_generator = test_datagen.flow_from_directory(
validationDir,
target_size=(img_width, img_height),
batch_size=32,
color_mode='grayscale',
class_mode='categorical')
model.fit_generator(
train_generator,
samples_per_epoch=nb_train_samples,
nb_epoch=nb_epoch,
validation_data=validation_generator,
nb_val_samples=nb_validation_samples)
return model
def try_params( n_iterations, params, data=None, datamode='memory'):
print "iterations:", n_iterations
print_params( params )
batchsize = 100
if datamode == 'memory':
X_train, Y_train = data['train']
X_valid, Y_valid = data['valid']
inputshape = X_train.shape[1:]
else:
train_generator = data['train']['gen_func'](batchsize, data['train']['path'])
valid_generator = data['valid']['gen_func'](batchsize, data['valid']['path'])
train_epoch_step = data['train']['n_sample'] / batchsize
valid_epoch_step = data['valid']['n_sample'] / batchsize
inputshape = data['train']['gen_func'](batchsize, data['train']['path']).next()[0].shape[1:]
model = Sequential()
model.add(Conv2D(128, (1, 24), padding='same', input_shape=inputshape, activation='relu'))
model.add(GlobalMaxPooling2D())
model.add(Dense(32,activation='relu'))
model.add(Dropout(params['DROPOUT']))
model.add(Dense(2))
model.add(Activation('softmax'))
optim = Adadelta
myoptimizer = optim(epsilon=params['DELTA'], rho=params['MOMENT'])
mylossfunc = 'categorical_crossentropy'
model.compile(loss=mylossfunc, optimizer=myoptimizer,metrics=['accuracy'])
early_stopping = EarlyStopping( monitor = 'val_loss', patience = 3, verbose = 0 )
if datamode == 'memory':
model.fit(
X_train,
Y_train,
batch_size=batchsize,
epochs=int( round( n_iterations )),
validation_data=(X_valid, Y_valid),
callbacks = [ early_stopping ])
score, acc = model.evaluate(X_valid,Y_valid)
else:
model.fit_generator(
train_generator,
steps_per_epoch=train_epoch_step,
epochs=int( round( n_iterations )),
validation_data=valid_generator,
validation_steps=valid_epoch_step,
callbacks = [ early_stopping ])
score, acc = model.evaluate_generator(valid_generator, steps=valid_epoch_step)
return { 'loss': score, 'model': (model.to_json(), optim, myoptimizer.get_config(), mylossfunc) }
def lm():
maxlen=10
cfig = getattr(config, 'get_config_morph')('cs')
batch_size, nb_epoch = cfig['batch_size'], 200
X_train, y_train = getTextFile(cfig['train_file'], cfig['train_dic'], cfig)
X_train = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post')
y_train = sequence.pad_sequences(y_train, maxlen=maxlen, padding='post')
#X_train, y_train = X_train[:10050], y_train[:10050]
print X_train.shape, y_train.shape
#X_test, y_test = getTextFile(cfig['test_file'], cfig['train_dic'], cfig)
#X_test = sequence.pad_sequences(X_test, maxlen=10, padding='post')
#y_test = sequence.pad_sequences(y_test, maxlen=10, padding='post')
'''
y_train_tensor3 = np.zeros((len(X_train), maxlen, cfig['vocab_size']), dtype=np.bool)
i, t = 0, 0
for sentence in y_train:
t = 0
for v in sentence:
y_train_tensor3[i][t][v] = True
t += 1
i += 1
k = 0
for i , j in generate_data(X_train, y_train, 200, cfig['vocab_size']):
print i.shape , j.shape
if k > 20:
break
k += 1
exit(0)
'''
print 'Build model...'
model = Sequential()
model.add(Embedding(cfig['vocab_size'], 128, dropout=0.2))
model.add(LSTM(128, return_sequences=True)) #- original
model.add(TimeDistributedDense(cfig['vocab_size']))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.summary()
print 'Train...'
model.fit_generator(generate_data(X_train, y_train, batch_size, cfig['vocab_size']),
#samples_per_epoch=len(X_train)/batch_size,
samples_per_epoch=1000,
nb_epoch=nb_epoch)
#model.fit(X_train, y_train_tensor3)
exit(0)
cnt = 0
for i , j in generate_data(X_train, y_train, 200, cfig['vocab_size']):
#model.train_on_batch(i, j)
history= model.fit(i, j, batch_size=10, nb_epoch=1,verbose=0)
if cnt >= 3:
break
cnt += 1
def train_model(genre, dir_model, MP):
sess = tf.Session(config=tf.ConfigProto(log_device_placement=True)) #check gpu is being used
batch_size = MP['bs']
lstm_size = MP['lstm_size']
seq_length = MP['seq_length']
drop = MP['dropout']
lr = MP['lr']
epochs = MP['epochs']
text_to_int, int_to_text, n_chars = np.load('playlists/%s/ancillary_char.npy'%genre)
vocab_size = len(text_to_int)
X = np.load('playlists/%s/X_sl%d_char.npy'%(genre, seq_length))
y = np.load('playlists/%s/y_sl%d_char.npy'%(genre, seq_length))
# randomly shuffle samples before test/valid split
np.random.seed(40)
ran = [i for i in range(len(X))]
np.random.shuffle(ran)
X_train, X_valid, y_train, y_valid = train_test_split(X[ran], y[ran], test_size=0.2, random_state=42)
try:
model = load_model(dir_model)
print("successfully loaded previous model, continuing to train")
except:
print("generating new model")
model = Sequential()
model.add(GRU(lstm_size, dropout=drop, recurrent_dropout=drop, return_sequences=True,
input_shape=(seq_length, vocab_size)))
for i in range(MP['n_layers'] - 1):
model.add(GRU(lstm_size, dropout=drop, recurrent_dropout=drop, return_sequences=True))
model.add(TimeDistributed(Dense(vocab_size, activation='softmax'))) #output shape=(bs, sl, vocab)
decay = 0.5*lr/epochs
optimizer = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=decay, clipvalue=1)
#optimizer = RMSprop(lr=lr, decay=decay)
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['categorical_accuracy'])
print(model.summary())
# callbacks
checkpoint = ModelCheckpoint(dir_model, monitor='loss', save_best_only=True, mode='min')
#earlystop = EarlyStopping(monitor='val_loss', min_delta=0.01, patience=3)
callbacks_list = [checkpoint]
# train
model.fit_generator(one_hot_gen(X_train, y_train, vocab_size, seq_length, batch_size),
steps_per_epoch=len(X_train)/batch_size, epochs=epochs, callbacks=callbacks_list,
validation_data=one_hot_gen(X_valid, y_valid, vocab_size, seq_length, batch_size),
validation_steps=len(X_valid)/batch_size)
model.save(dir_model)
def test_sequential_fit_generator():
(X_train, y_train), (X_test, y_test) = _get_test_data()
def data_generator(train):
if train:
max_batch_index = len(X_train) // batch_size
else:
max_batch_index = len(X_test) // batch_size
i = 0
while 1:
if train:
yield (X_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size])
else:
yield (X_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size])
i += 1
i = i % max_batch_index
model = Sequential()
model.add(Dense(nb_hidden, input_shape=(input_dim,)))
model.add(Activation('relu'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit_generator(data_generator(True), len(X_train), nb_epoch, show_accuracy=False)
model.fit_generator(data_generator(True), len(X_train), nb_epoch, show_accuracy=True)
model.fit_generator(data_generator(True), len(X_train), nb_epoch, show_accuracy=False, validation_data=(X_test, y_test))
model.fit_generator(data_generator(True), len(X_train), nb_epoch, show_accuracy=True, validation_data=(X_test, y_test))
loss = model.evaluate(X_train, y_train, verbose=0)
assert(loss < 0.9)
def test_sequential_fit_generator():
(x_train, y_train), (x_test, y_test) = _get_test_data()
def data_generator(train):
if train:
max_batch_index = len(x_train) // batch_size
else:
max_batch_index = len(x_test) // batch_size
i = 0
while 1:
if train:
yield (x_train[i * batch_size: (i + 1) * batch_size], y_train[i * batch_size: (i + 1) * batch_size])
else:
yield (x_test[i * batch_size: (i + 1) * batch_size], y_test[i * batch_size: (i + 1) * batch_size])
i += 1
i = i % max_batch_index
model = Sequential()
model.add(Dense(num_hidden, input_shape=(input_dim,)))
model.add(Activation('relu'))
model.add(Dense(num_class))
model.pop()
model.add(Dense(num_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit_generator(data_generator(True), 5, epochs)
model.fit_generator(data_generator(True), 5, epochs,
validation_data=(x_test, y_test))
model.fit_generator(data_generator(True), 5, epochs,
validation_data=data_generator(False),
validation_steps=3)
model.fit_generator(data_generator(True), 5, epochs, max_queue_size=2)
model.evaluate(x_train, y_train)
def model(datagen, X_train, Y_train, X_test, Y_test):
batch_size = 32
nb_epoch = 200
# input image dimensions
img_rows, img_cols = 32, 32
# the CIFAR10 images are RGB
img_channels = 3
model = Sequential()
model.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Flatten())
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# fit the model on the batches generated by datagen.flow()
model.fit_generator(datagen.flow(X_train, Y_train,
batch_size=batch_size),
samples_per_epoch=X_train.shape[0],
nb_epoch=nb_epoch,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def test_multiprocessing_training_fromfile():
reached_end = False
arr_data = np.random.randint(0,256, (500, 200))
arr_labels = np.random.randint(0, 2, 500)
np.savez("data.npz", **{"data": arr_data, "labels": arr_labels})
def myGenerator():
batch_size = 32
n_samples = 500
arr = np.load("data.npz")
while True:
batch_index = np.random.randint(0, n_samples - batch_size)
start = batch_index
end = start + batch_size
X = arr["data"][start: end]
y = arr["labels"][start: end]
yield X, y
# Build a NN
model = Sequential()
model.add(Dense(10, input_shape=(200, )))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('linear'))
model.compile(loss='mse', optimizer='adadelta')
model.fit_generator(myGenerator(),
samples_per_epoch=320,
nb_epoch=1,
verbose=1,
max_q_size=10,
nb_worker=2,
pickle_safe=True)
model.fit_generator(myGenerator(),
samples_per_epoch=320,
nb_epoch=1,
verbose=1,
max_q_size=10,
pickle_safe=False)
reached_end = True
assert reached_end
def test_TerminateOnNaN():
np.random.seed(1337)
(X_train, y_train), (X_test, y_test) = get_data_callbacks()
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
cbks = [callbacks.TerminateOnNaN()]
model = Sequential()
initializer = initializers.Constant(value=1e5)
for _ in range(5):
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu',
kernel_initializer=initializer))
model.add(Dense(num_classes, activation='linear'))
model.compile(loss='mean_squared_error',
optimizer='rmsprop')
# case 1 fit
history = model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=cbks, epochs=20)
loss = history.history['loss']
assert len(loss) == 1
assert loss[0] == np.inf
history = model.fit_generator(data_generator(X_train, y_train, batch_size),
len(X_train),
validation_data=(X_test, y_test),
callbacks=cbks,
epochs=20)
loss = history.history['loss']
assert len(loss) == 1
assert loss[0] == np.inf or np.isnan(loss[0])
def Convolution(test, df, flip_indices):
img_rows, img_cols = 96, 96 # images are 96x96 pixels
img_channels = 1 # images are grey scale - if RGB use img_channels = 3
nb_filter = 32 # common and efficient to use multiples of 2 for filters
nb_epoch = 1
batch_size = 32
X, y = load(df, test)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=99)
model = Sequential()
model.add(Convolution2D(nb_filter = nb_filter, nb_row = 3, nb_col = 3,
# border_mode = 'same', # this causes a crash, even though default is 'same'....
input_shape = X_train.shape[1:])) # must set input shape for first call to Convolutional2D
model.add(Activation('relu')) # default convolutional activation function - should try Leaky ReLU and PReLU functions : http://arxiv.org/abs/1502.01852
model.add(MaxPooling2D()) # pooling 2x2 with stride 2 is default - reduces size of matrix by half in both dimensions
# There is a new paper quesitoning the use of max pooling - finding that replacement with convolutional layers with increased stride works better
model.add(Dropout(0.1))
model.add(Convolution2D(nb_filter = nb_filter * 2, nb_row = 2, nb_col = 2))
model.add(Activation('relu'))
model.add(MaxPooling2D())
model.add(Dropout(0.2))
model.add(Convolution2D(nb_filter = nb_filter * 4, nb_row = 2, nb_col = 2))
model.add(Activation('relu'))
model.add(MaxPooling2D())
model.add(Dropout(0.3))
model.add(Flatten()) # flatten the data before fully connected layer (reg. neural net)
model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.4))
model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.5))
if len(flip_indices) == 12:
model.add(Dense(30))
else:
model.add(Dense(8))
model.compile(loss = 'mean_squared_error', # other objectives : http://keras.io/objectives/
optimizer = 'Adam') # discussion : http://cs231n.github.io/neural-networks-3/ : "Adam is currently recommended as the default algorithm to use" - cs231n
dataGen = createDataGen(flip_indices = flip_indices, horizontal_flip = True)
checkpointer = ModelCheckpoint(filepath="/tmp/weights.hdf5", verbose=1, save_best_only=True)
earlystopping = EarlyStopping(monitor = 'val_loss', patience = 20, verbose = 0, mode = 'auto')
model.fit_generator(dataGen.flow(X_train, y_train, batch_size = batch_size), nb_epoch = nb_epoch,
samples_per_epoch = X_train.shape[0],
validation_data = (X_test, y_test),
callbacks = [checkpointer, earlystopping])
class TrainAllCnn(object):
def __init__(self):
# prepare trainigs data
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
self.Y_train = np_utils.to_categorical(y_train, 10)
self.Y_test = np_utils.to_categorical(y_test, 10)
self.X_train = X_train.astype('float32')
self.X_test = X_test.astype('float32')
self.X_train /= 255
self.X_test /= 255
def build_model(self):
# we want an sequential model
self.model = Sequential()
self.model.add(Convolution2D(96, 3, 3, border_mode='same', input_shape=(3, 32, 32)))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(96, 3, 3, border_mode='same'))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(96, 3, 3, subsample=(2,2), border_mode='same'))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(192, 3, 3, border_mode='same'))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(192, 3, 3, border_mode='same'))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(96, 3, 3, subsample=(2, 2), border_mode='same'))
self.model.add(Activation('relu'))
self.model.add(MaxPooling2D(pool_size=(3, 3), strides=(2,2)))
self.model.add(Convolution2D(192, 3, 3))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(192, 1, 1))
self.model.add(Activation('relu'))
self.model.add(Convolution2D(10, 1, 1))
self.model.add(Activation('relu'))
self.model.add(Flatten())
self.model.add(Activation('softmax'))
# now we compile the model, asoptimizer we use stochastik gradient decent with momentum
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
self.model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
def train(self, batch, epoch):
self.build_model()
# now we fit the model
# with 200 epochs this takes a while ...
return self.model.fit(self.X_train, self.Y_train, batch_size=batch, nb_epoch=epoch, validation_data=(self.X_test, self.Y_test), shuffle=True)
def train_argument_images(self, batch, epoch):
self.build_model()
datagen = ImageDataGenerator(featurewise_center=False, samplewise_center=False, featurewise_std_normalization=False, samplewise_std_normalization=False, zca_whitening=False, rotation_range=0, width_shift_range=0.1, height_shift_range=0.1, horizontal_flip=True, vertical_flip=False, zoom_range=0.1)
datagen.fit(self.X_train)
return self.model.fit_generator(datagen.flow(self.X_train, self.Y_train, batch_size=batch), samples_per_epoch=self.X_train.shape[0], nb_epoch=epoch, validation_data=(self.X_test, self.Y_test)).model
def main():
ext = extension_from_parameters()
out_dim = 1
loss = 'mse'
metrics = None
#metrics = ['accuracy'] if CATEGORICAL else None
reshape = LOCALLY_CONNECTED_LAYERS is not None
datagen = RegressionDataGenerator()
train_gen = datagen.flow(batch_size=BATCH_SIZE, reshape=reshape)
val_gen = datagen.flow(val=True, batch_size=BATCH_SIZE, reshape=reshape)
val_gen2 = datagen.flow(val=True, batch_size=BATCH_SIZE, reshape=reshape)
model = Sequential()
if LOCALLY_CONNECTED_LAYERS:
for layer in LOCALLY_CONNECTED_LAYERS:
if layer:
model.add(LocallyConnected1D(*layer, input_shape=(datagen.input_dim, 1), activation=ACTIVATION))
if POOL:
model.add(MaxPooling1D(pool_length=POOL))
model.add(Flatten())
for layer in DENSE_LAYERS:
if layer:
model.add(Dense(layer, input_dim=datagen.input_dim, activation=ACTIVATION))
if DROP:
model.add(Dropout(DROP))
model.add(Dense(out_dim))
model.summary()
model.compile(loss=loss, optimizer='sgd', metrics=metrics)
train_samples = int(datagen.n_train/BATCH_SIZE) * BATCH_SIZE
val_samples = int(datagen.n_val/BATCH_SIZE) * BATCH_SIZE
history = BestLossHistory(val_gen2, val_samples, ext)
checkpointer = ModelCheckpoint(filepath='model'+ext+'.h5', save_best_only=True)
model.fit_generator(train_gen, train_samples,
nb_epoch = NB_EPOCH,
validation_data = val_gen,
nb_val_samples = val_samples,
callbacks=[history, checkpointer])
def test_CallbackValData():
np.random.seed(1337)
(X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim,),
classification=True,
num_classes=num_class)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
model = Sequential()
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_class, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
cbk = callbacks.LambdaCallback(on_train_end=lambda x: 1)
model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=[cbk], epochs=1)
def data_generator(train):
if train:
max_batch_index = len(X_train) // batch_size
else:
max_batch_index = len(X_test) // batch_size
i = 0
while 1:
if train:
yield (X_train[i * batch_size: (i + 1) * batch_size],
y_train[i * batch_size: (i + 1) * batch_size])
else:
yield (X_test[i * batch_size: (i + 1) * batch_size],
y_test[i * batch_size: (i + 1) * batch_size])
i += 1
i = i % max_batch_index
cbk2 = callbacks.LambdaCallback(on_train_end=lambda x: 1)
model.fit_generator(data_generator(True), len(X_train), epochs=1,
validation_data=(X_test, y_test),
callbacks=[cbk2])
# callback validation data should always have x, y, and sample weights
assert len(cbk.validation_data) == len(cbk2.validation_data) == 3
assert cbk.validation_data[0] is cbk2.validation_data[0]
assert cbk.validation_data[1] is cbk2.validation_data[1]
assert cbk.validation_data[2].shape == cbk2.validation_data[2].shape
def test_multiprocessing_training():
reached_end = False
arr_data = np.random.randint(0, 256, (500, 2))
arr_labels = np.random.randint(0, 2, 500)
def myGenerator():
batch_size = 32
n_samples = 500
while True:
batch_index = np.random.randint(0, n_samples - batch_size)
start = batch_index
end = start + batch_size
X = arr_data[start: end]
y = arr_labels[start: end]
yield X, y
# Build a NN
model = Sequential()
model.add(Dense(1, input_shape=(2, )))
model.compile(loss='mse', optimizer='adadelta')
model.fit_generator(myGenerator(),
samples_per_epoch=320,
nb_epoch=1,
verbose=1,
max_q_size=10,
nb_worker=4,
pickle_safe=True)
model.fit_generator(myGenerator(),
samples_per_epoch=320,
nb_epoch=1,
verbose=1,
max_q_size=10,
pickle_safe=False)
reached_end = True
assert reached_end
def train(train_generator,train_size,input_num,dims_num):
print("Start Train Job! ")
start=time.time()
inputs=InputLayer(input_shape=(input_num,dims_num),batch_size=batch_size)
layer1=LSTM(128)
output=Dense(2,activation="softmax",name="Output")
optimizer=Adam()
model=Sequential()
model.add(inputs)
model.add(layer1)
model.add(Dropout(0.5))
model.add(output)
call=TensorBoard(log_dir=log_dir,write_grads=True,histogram_freq=1)
model.compile(optimizer,loss="categorical_crossentropy",metrics=["accuracy"])
model.fit_generator(train_generator,steps_per_epoch=train_size//batch_size,epochs=epochs_num,callbacks=[call])
# model.fit_generator(train_generator, steps_per_epoch=5, epochs=5, callbacks=[call])
model.save(model_dir)
end=time.time()
print("Over train job in %f s"%(end-start))
class SequenceTaggingMachine(object):
def __init__(self, mask ):
self.model = None
self.mask = mask
def train(self, train_corpus, valid_corpus, learning_param):
self.model = Sequential()
self.model.add(Embedding(train_corpus.source_cell_num(), 256, mask_zero = True))
self.model.add(Bidirectional(LSTM(128, return_sequences=True)))
self.model.add(Bidirectional(LSTM(128, return_sequences=True)))
self.model.add(TimeDistributed(Dense(input_dim=128, output_dim=train_corpus.target_cell_num())))
self.model.add(Activation('softmax'))
self.model.compile(loss=lambda output,target: masked_categorical_crossentropy(output, target, self.mask),
optimizer='rmsprop',
metrics=[lambda y_true, y_pred: masked_categorical_accuracy(y_true, y_pred, self.mask)])
logging.debug("Preparing data iter")
problem = SequenceTaggingProblem(train_corpus)
data_train = BucketIter(problem, learning_param.batch_size, max_pad_num=learning_param.max_pad)
val_problem = SequenceTaggingProblem(valid_corpus)
data_val = BucketIter(val_problem, learning_param.batch_size, max_pad_num=learning_param.max_pad)
checkpointer = ModelCheckpoint(filepath="weights.{epoch:03d}-{val_loss:.2f}.hdf5", verbose=1)
logging.debug("Begin train model")
self.model.fit_generator(bucket_iter_adapter(data_train,train_corpus.target_cell_num()),
samples_per_epoch=train_corpus.corpus_size(), nb_epoch=100, verbose=1,
validation_data=bucket_iter_adapter(data_val, train_corpus.target_cell_num()),
nb_val_samples = valid_corpus.corpus_size(), callbacks=[checkpointer])
print "Model is trained"
def test_multiprocessing_training_fromfile(in_tmpdir):
arr_data = np.random.randint(0, 256, (50, 2))
arr_labels = np.random.randint(0, 2, 50)
np.savez('data.npz', **{'data': arr_data, 'labels': arr_labels})
def custom_generator():
batch_size = 10
n_samples = 50
arr = np.load('data.npz')
while True:
batch_index = np.random.randint(0, n_samples - batch_size)
start = batch_index
end = start + batch_size
X = arr['data'][start: end]
y = arr['labels'][start: end]
yield X, y
# Build a NN
model = Sequential()
model.add(Dense(1, input_shape=(2, )))
model.compile(loss='mse', optimizer='adadelta')
model.fit_generator(custom_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
workers=2,
use_multiprocessing=True)
model.fit_generator(custom_generator(),
steps_per_epoch=5,
epochs=1,
verbose=1,
max_queue_size=10,
use_multiprocessing=False)
os.remove('data.npz')
def rnn_gru(float_data, lookback=1440, step=6):
model = Sequential()
model.add(layers.GRU(32, input_shape=(None, float_data.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
history = model.fit_generator(train_gen,
steps_per_epoch=500,
epochs=20,
validation_data=val_gen,
validation_steps=val_steps)
draw_histroy(history)
def test_multithreading_from_file():
arr_data = np.random.randint(0, 256, (50, 2))
arr_labels = np.random.randint(0, 2, 50)
np.savez('data_threads.npz', **{'data': arr_data, 'labels': arr_labels})
@threadsafe_generator
def custom_generator():
batch_size = 10
n_samples = 50
arr = np.load('data_threads.npz')
while True:
batch_index = np.random.randint(0, n_samples - batch_size)
start = batch_index
end = start + batch_size
X = arr['data'][start: end]
y = arr['labels'][start: end]
yield X, y
# Build a NN
model = Sequential()
model.add(Dense(1, input_shape=(2,)))
model.compile(loss='mse', optimizer='adadelta')
# - Produce data on 4 worker threads, consume on main thread:
# - All worker threads share the SAME generator
model.fit_generator(custom_generator(),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=1,
verbose=1,
validation_steps=None,
max_queue_size=10,
workers=WORKERS,
use_multiprocessing=False)
# - Produce data on 1 worker thread, consume on main thread:
# - Worker thread is the only thread running the generator
model.fit_generator(custom_generator(),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=1,
verbose=1,
validation_steps=None,
max_queue_size=10,
workers=1,
use_multiprocessing=False)
# - Produce and consume data without a queue on main thread
# - Make sure the value of `use_multiprocessing` is ignored
model.fit_generator(custom_generator(),
steps_per_epoch=STEPS_PER_EPOCH,
epochs=1,
verbose=1,
validation_steps=None,
max_queue_size=10,
workers=0,
use_multiprocessing=False)
os.remove('data_threads.npz')
def train_xy(epochs=50, batch_size=32, h=256, w=256, ch=3, train_p=0.8, valid_p=0.1):
print("Compiling Model")
t_comp = time()
model = Sequential()
# reshape input to ch, h, w (no sample axis)
model.add(Reshape(dims=(h, w, ch), input_shape=(ch * h * w,)))
model.add(Permute((3, 1, 2)))
# add conv layers
model.add(Convolution2D(16, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(output_dim=2000, init="glorot_uniform", activation="relu", W_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(output_dim=2000, init="glorot_uniform", activation="relu", W_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(output_dim=4, init="glorot_uniform", activation="relu"))
model.compile(optimizer="rmsprop", loss="mse")
t_train = time()
print("Took %.1fs" % (t_train - t_comp))
# split dataset
i_test = int(train_p * nrow) / batch_size * batch_size
i_valid = int(i_test * (1 - valid_p)) / batch_size * batch_size
X_train, Y_train = X_[:i_valid,], Y_[:i_valid,]
X_valid, Y_valid = X_[i_valid:i_test,], Y_[i_valid:i_test,]
# naive fitting to lower rmse faster
hist = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10, verbose=1, validation_split=0.1)
print(hist)
# fit by batch using generator!
img_aug = image_augmentor(X_, Y_, i_valid)
hist = model.fit_generator(
generator=img_aug,
samples_per_epoch=i_valid,
nb_epoch=5000,
verbose=1,
validation_data=(X_valid, Y_valid),
nb_worker=1,
)
rmse_test = model.evaluate(X_[i_test:,], Y_[i_test:,])
print("Test RMSE: %.4f" % rmse_test)
# save model
model_json = model.to_json()
open(path_img + "locate/model_116.json", "w").write(model_json)
model.save_weights(path_img + "locate/model_116_weights.h5")
def test_stop_training_csv(tmpdir):
np.random.seed(1337)
fp = str(tmpdir / 'test.csv')
(X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim,),
classification=True,
num_classes=num_classes)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
cbks = [callbacks.TerminateOnNaN(), callbacks.CSVLogger(fp)]
model = Sequential()
for _ in range(5):
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_classes, activation='linear'))
model.compile(loss='mean_squared_error',
optimizer='rmsprop')
def data_generator():
i = 0
max_batch_index = len(X_train) // batch_size
tot = 0
while 1:
if tot > 3 * len(X_train):
yield (np.ones([batch_size, input_dim]) * np.nan,
np.ones([batch_size, num_classes]) * np.nan)
else:
yield (X_train[i * batch_size: (i + 1) * batch_size],
y_train[i * batch_size: (i + 1) * batch_size])
i += 1
tot += 1
i = i % max_batch_index
history = model.fit_generator(data_generator(),
len(X_train) // batch_size,
validation_data=(X_test, y_test),
callbacks=cbks,
epochs=20)
loss = history.history['loss']
assert len(loss) > 1
assert loss[-1] == np.inf or np.isnan(loss[-1])
values = []
with open(fp) as f:
for x in reader(f):
values.append(x)
assert 'nan' in values[-1], 'The last epoch was not logged.'
os.remove(fp)
