def build_and_train(self, x, y):
from keras.layers import Layer, Dense, Conv1D, MaxPool1D, pooling, MaxPool1D, Flatten
from keras.models import Sequential
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()
onehot_y = encoder.fit_transform(y)
lx,ly,lz = x.shape
self.meta.update({
"word_vec_size": lz,
"len_words": ly,
"encoder_classes": encoder.classes_
})
model = Sequential()
model.add(Conv1D(2, 3, input_shape=(ly, lz), activation='sigmoid'))
# model.add(MaxPool1D())
model.add(Flatten())
model.add(Dense(3, activation='sigmoid'))
model.add(Dense(onehot_y.shape[1], activation='softmax'))
model.compile('Adam', loss='mse')
model.fit(x, onehot_y, epochs=100)
score = float(model.test_on_batch(x, onehot_y))
self.meta['score'] = score
self.set_status_message(f'Model fit complete. Score {score}')
return model
def NN_Basic(dataX, dataY, labels, look_back=10):
dataX, dataY, labels = ppu.combine_history(dataX,
dataY,
labels,
look_back=look_back)
dataX = (dataX - np.min(dataX)) / (np.max(dataX) - np.min(dataX))
input_dim = dataX.shape[1]
model = Sequential()
model.add(Dense(64, activation='relu', input_dim=input_dim))
model.add(Dropout(0.2))
model.add(Dense(16, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model.fit(dataX, labels * 1.0, epochs=1000, batch_size=8)
print "Neural Network Basic Accuracy:" + str(
model.test_on_batch(dataX, labels)[1])
plt.plot(range(1000), history.history['acc'])
plt.show()
return model
def main():
if len(sys.argv) <= 1:
print("Usage: {0} [pointers/training.txt]".format(sys.argv[0]))
return
pointers_file = sys.argv[1]
dataset, params = load(
pointers_file)[::6] ## first & last of ((7-1) returned variables)
print(str(len(dataset)) + " " + str(len(params)))
print(params)
print("Intitializing neural network...")
model = Sequential()
model.add(Dropout(0.1, input_shape=(7500, )))
model.add(Dense(7500))
model.add(Activation('tanh'))
model.add(Dropout(0.05))
model.add(Dense(1875))
model.add(Activation('tanh'))
model.add(Dense(512))
model.add(Activation('tanh'))
model.add(Dropout(0.01))
model.add(Dense(128))
model.add(Activation('tanh'))
model.add(Dense(4)) # output // 4 categories
model.add(Activation('softmax'))
''' model.add(Dense(8, input_dim=7500))
model.add(Activation('tanh'))
model.add(Dense(4)) # output // 4 categories
model.add(Activation('softmax')) '''
adadelta = Adadelta(lr=0.01)
adam = Adam(lr=0.01)
sgd = SGD(lr=0.01)
model.compile(loss='binary_crossentropy',
optimizer=sgd) #, metrics=['accuracy'])
#model.compile(loss='categorical_crossentropy', optimizer=sgd)#, metrics=['categorical_accuracy'])
print("Training neural network...")
model.fit(np.array(dataset),
np.array(params),
verbose=1,
batch_size=1,
epochs=5) #, validation_split=0.3, shuffle=True)
testx = np.array(dataset[0:1])
testy = np.array(params[0:1])
testx2 = np.array([dataset[-1]])
testx3 = np.array([dataset[8]])
print(model.predict_on_batch(np.array(dataset)))
print(model.predict(np.array(dataset)))
print(model.predict_classes(np.array(dataset)))
score = model.evaluate(testx, testy)
print(score)
print(model.test_on_batch(testx, testy))
def BuildModelKeras(self, test_len, error_tol):
start = time.time()
model = Sequential()
# build a LSTM RNN
# model.add(ResidualKeras(units=self.n_hidden, input_shape=(self.n_steps, self.n_input)))
model.add(
LSTM(units=self.n_hidden,
input_shape=(self.n_steps, self.n_input)))
model.add(Dense(self.n_classes))
# compile
adam = Adam(self.learning_rate)
model.compile(optimizer=adam, loss='mean_squared_error')
# data
no_of_samples = np.shape(self.X)[0]
test_data = self.X[no_of_samples - test_len:].reshape(
(-1, self.n_steps, self.n_input))
test_label = self.Y[no_of_samples - test_len:].reshape(
(-1, self.n_classes))
step = 1
cost = error_tol
Loss = []
Elapsed = []
while ((step < self.training_iters) & (cost >= error_tol)):
index = np.random.random_integers(0, no_of_samples - test_len,
self.batch_size)
batch_x = self.X[index, :]
batch_y = self.Y[index].reshape((self.batch_size, 1))
batch_x = batch_x.reshape(
(self.batch_size, self.n_steps, self.n_input))
cost = model.train_on_batch(batch_x, batch_y)
pred = model.predict(batch_x, self.batch_size)
print("Iter " + str(step) + ", Training Accuracy= " +
"{:.7f}".format(cost))
if step % self.display_step == 0:
Loss.append(cost)
Elapsed.append((time.time() - start) / 3600)
print("Iter " + str(step) + ", Training Accuracy= " +
"{:.7f}".format(cost))
step += 1
"""
train_x = self.X.reshape((2000, self.n_steps, self.n_input))
train_y = self.Y.reshape((2000,1))
model.fit(train_x, train_y, epochs=5000, batch_size=self.batch_size)
"""
model.summary()
# Calculate accuracy for test data
result = model.test_on_batch(test_data, test_label)
# Save the model
model.save('Kerasmodel.h5')
print("Testing Loss:", result)
elapsed = time.time() - start
"""
if self.deg_of_sig==0:
plt.plot(Elapsed, Loss, label='Time series')
else:
plt.plot(Elapsed, Loss, label='degree %d' %self.deg_of_sig)
"""
return {'Loss': result, 'Time': elapsed, 'NStep': step}
def BuildModelKeras(self, test_len, error_tol):
start = time.time()
model = Sequential()
# build a LSTM RNN
# model.add(ResidualKeras(units=self.n_hidden, input_shape=(self.n_steps, self.n_input)))
model.add(
LSTM(units=self.n_hidden,
input_shape=(self.n_steps, self.n_input)))
model.add(Dense(self.n_classes))
# compile
adam = Adam(self.learning_rate)
model.compile(optimizer=adam, loss='mean_squared_error')
# data
no_of_samples = np.shape(self.X)[0]
test_data = self.X[no_of_samples - test_len:].reshape(
(-1, self.n_steps, self.n_input))
test_label = self.Y[no_of_samples - test_len:].reshape(
(-1, self.n_classes))
step = 1
cost = error_tol
Loss = []
Elapsed = []
while ((step < self.training_iters) & (cost >= error_tol)):
index = np.random.random_integers(0, no_of_samples - test_len,
self.batch_size)
batch_x = self.X[index, :]
batch_y = self.Y[index].reshape((self.batch_size, 1))
batch_x = batch_x.reshape(
(self.batch_size, self.n_steps, self.n_input))
cost = model.train_on_batch(batch_x, batch_y)
pred = model.predict(batch_x, self.batch_size)
if step % self.display_step == 0:
Loss.append(cost)
Elapsed.append((time.time() - start) / 3600)
print("Iter " + str(step) + ", Training Accuracy= " +
"{:.7f}".format(cost))
step += 1
model.summary()
# Calculate accuracy for test data
result = model.test_on_batch(test_data, test_label)
# Save the model
model.save(self.prefix + 'model_logsig%d_segment%d.h5' %
(self.deg_of_logsig, self.n_steps))
elapsed = time.time() - start
test_pred = model.predict(test_data)
return {
'Loss': result,
'Time': elapsed,
'Pred': test_pred,
'model': model
}
def build_and_train(self, x, y):
from keras.layers import Dense, Activation, Dropout, Flatten
from keras.models import Sequential
from sklearn.preprocessing import LabelBinarizer
import matplotlib.pyplot as plt
encoder = LabelBinarizer()
onehot_y = encoder.fit_transform(y)
lx, ly, lz = x.shape
self.meta.update({
"word_vec_size": lz,
"len_words": ly,
"encoder_classes": encoder.classes_
})
model = Sequential()
#много слоев
model.add(Dense(120, input_shape=(ly, lz)))
model.add(Dense(160, activation='tanh'))
model.add(Dropout(0.1))
model.add(Dense(100, activation='sigmoid'))
model.add(Dropout(0.2))
model.add(Dense(60, activation='relu'))
model.add(Dropout(0.3))
model.add(Dense(80, activation='hard_sigmoid'))
model.add(Flatten())
model.add(Dense(onehot_y.shape[1], activation='softmax'))
model.compile('Adam', loss='mse', metrics=['acc'])
history = model.fit(x,
onehot_y,
batch_size=5,
epochs=100,
validation_split=0.1)
scores = float(model.test_on_batch(x, onehot_y)[0])
print(f'точность на тестовых данных: {scores*100}')
plt.figure(0)
plt.subplot(2, 1, 1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'], '--')
plt.title('Loss')
plt.legend(['Loss', 'Val loss'])
plt.subplot(2, 1, 2)
plt.title('Acc')
plt.plot(history.history['acc'])
plt.savefig(str(Path(self.output().path).with_suffix('.png')))
return model
def build_and_train(self, x, y):
from keras.layers import Dense, Activation, Dropout, Flatten
from keras.models import Sequential
from sklearn.preprocessing import LabelBinarizer
#from keras.callbacks import ModelCheckpoint
import matplotlib.pyplot as plt
encoder = LabelBinarizer()
onehot_y = encoder.fit_transform(y)
lx, ly, lz = x.shape
self.meta.update({
"word_vec_size": lz,
"len_words": ly,
"encoder_classes": encoder.classes_
})
model = Sequential()
#+drop 0.1
model.add(Dense(60, input_shape=(ly, lz)))
model.add(Dropout(0.1))
#activate fun exp
model.add(Dense(60, activation='relu'))
#drop +0.1
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(onehot_y.shape[1], activation='softmax'))
model.compile('Adam', loss='mse', metrics=['acc'])
#checkpointer = ModelCheckpoint(filepath='C:/Users/dkkay/Desktop/Дипломная/train-nn-dev/dist/checkpoints', verbose=1, save_best_only=True)
history = model.fit(x,
onehot_y,
batch_size=5,
epochs=100,
validation_split=0) #callbacks=[checkpointer])
scores = float(model.test_on_batch(x, onehot_y)[0])
print(f'точность на тестовых данных: {scores*100}')
plt.figure(0)
plt.subplot(2, 1, 1)
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'], '--')
plt.title('Loss')
plt.legend(['Loss', 'Val loss'])
plt.subplot(2, 1, 2)
plt.title('Acc')
plt.plot(history.history['acc'])
plt.savefig(str(Path(self.output().path).with_suffix('.png')))
return model
class LSTM_RNN:
def __init__(self,
look_back,
dropout_probability=0.2,
init='he_uniform',
loss='mse',
optimizer='rmsprop'):
self.rnn = Sequential()
self.look_back = look_back
self.rnn.add(
LSTM(10, stateful=True, batch_input_shape=(1, 1, 1), init=init))
self.rnn.add(Dropout(dropout_probability))
self.rnn.add(Dense(1, init=init))
self.rnn.compile(loss=loss, optimizer=optimizer)
def batch_train_test(self, trainX, trainY, testX, testY, nb_epoch=150):
print('Training LSTM-RNN...')
for epoch in range(nb_epoch):
print('Epoch ' + str(epoch + 1) + '/{}'.format(nb_epoch))
training_losses = []
testing_losses = []
for i in range(len(trainX)):
y_actual = trainY[i]
for j in range(self.look_back):
training_loss = self.rnn.train_on_batch(
np.expand_dims(np.expand_dims(trainX[i][j], axis=1),
axis=1), np.array([y_actual]))
training_losses.append(training_loss)
self.rnn.reset_states()
print('Mean training loss = {}'.format(np.mean(training_losses)))
mean_testing_loss = []
for i in range(len(testX)):
for j in range(self.look_back):
testing_loss = self.rnn.test_on_batch(
np.expand_dims(np.expand_dims(testX[i][j], axis=1),
axis=1), np.array([testY[i]]))
testing_losses.append(testing_loss)
self.rnn.reset_states()
for j in range(self.look_back):
y_pred = self.rnn.predict_on_batch(
np.expand_dims(np.expand_dims(testX[i][j], axis=1),
axis=1))
self.rnn.reset_states()
mean_testing_loss = np.mean(testing_losses)
print('Mean testing loss = {}'.format(mean_testing_loss))
return mean_testing_loss
def train_model(feature_layers,
classification_layers,
image_list,
nb_epoch,
nb_classes,
img_rows,
img_cols,
weights=None):
# Create testset data for cross-val
num_images = len(image_list)
test_size = int(0.2 * num_images)
print("Train size: ", num_images - test_size)
print("Test size: ", test_size)
model = Sequential()
for l in feature_layers + classification_layers:
model.add(l)
if not (weights is None):
model.set_weights(weights)
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
print('Using real time data augmentation')
for e in range(nb_epoch):
print('-' * 40)
print('Epoch', e)
print('-' * 40)
print('Training...')
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(num_images - test_size)
for X_batch, Y_batch in flow(image_list[0:-test_size]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('train loss', loss)])
print('Testing...')
# test time!
progbar = generic_utils.Progbar(test_size)
for X_batch, Y_batch in flow(image_list[-test_size:]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
score = model.test_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('test loss', score)])
return model, model.get_weights()
def test_model_learning_phase(self):
def one_zero(x):
return K.in_train_phase(K.zeros_like(x), K.ones_like(x))
model = Sequential([Lambda(one_zero, input_shape=(1, ))])
model.compile("sgd", "mse")
x = np.random.rand(10, 1)
l1 = model.test_on_batch(x, np.ones_like(x))
l2 = model.train_on_batch(x, np.zeros_like(x))
self.assertEqual(l1, 0)
self.assertEqual(l2, 0)
model = OracleWrapper(model, BiasedReweightingPolicy(), score="loss")
l1 = model.evaluate_batch(x, np.ones_like(x))[0].sum()
l2 = model.train_batch(x, np.zeros_like(x), np.ones_like(x))[0].sum()
self.assertEqual(l1, 0)
self.assertEqual(l2, 0)
def NN_LSTM(dataX, dataY, labels, look_back):
trainX = np.reshape(dataX, (dataX.shape[0], 1, dataX.shape[1]))
model = Sequential()
model.add(LSTM(input_dim=4, output_dim=16))
model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='sgd',
loss='binary_crossentropy',
metrics=['accuracy'])
history = model.fit(trainX, labels, epochs=1000, batch_size=look_back)
print "Neural Network LSTM Accuracy:" + str(
model.test_on_batch(trainX, labels)[1])
plt.plot(range(1000), history.history['acc'])
plt.show()
def train():
# model
X_shape = (FLAGS.image_height, FLAGS.image_width, 3)
model = Sequential()
model.add(Conv2D(128, kernel_size=3, padding=1, strides=1, input_shape=X_shape))
model.add(advanced_activations.LeakyReLU(alpha=0.3))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# conv => RELU => POOL
model.add(Conv2D(256, kernel_size=3, padding=1, strides=1))
model.add(advanced_activations.LeakyReLU(alpha=0.3))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
# flatten => RELU layers
model.add(Conv2D(512, kernel_size=3, padding=1, strides=1))
model.add(advanced_activations.LeakyReLU(alpha=0.3))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Conv2D(1024, kernel_size=3, padding=1, strides=1))
model.add(advanced_activations.LeakyReLU(alpha=0.3))
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2)))
model.add(Dense(FLAGS.charset_size)
model.add(Activation('softmax'))
optimizer = Adam()
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
for epoch in range(FLAGS.num_epochs):
for i, (X_train, y_train) in enumerate(train_data_iterator()):
X_train, y_train = np.array(X_train), np.array(y_train)
loss = model.train_on_batch(X_train, y_train)
print(f"epoch: {epoch} step: {i} loss: {loss}")
if i % 100 == 0:
X_test, y_test = test_data_helper()
X_test, y_test = np.array(X_test), np.array(y_test)
score = model.test_on_batch(X_test, y_test)
print(f"score: {score}")
if __name__ == '__main__':
train()
def train_model(feature_layers, classification_layers, image_list, nb_epoch, nb_classes, img_rows, img_cols, weights=None):
# Create testset data for cross-val
num_images = len(image_list)
test_size = int(0.2 * num_images)
print("Train size: ", num_images-test_size)
print("Test size: ", test_size)
model = Sequential()
for l in feature_layers + classification_layers:
model.add(l)
if not(weights is None):
model.set_weights(weights)
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
print('Using real time data augmentation')
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print('Training...')
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(num_images-test_size)
for X_batch, Y_batch in flow(image_list[0:-test_size]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('train loss', loss)])
print('Testing...')
# test time!
progbar = generic_utils.Progbar(test_size)
for X_batch, Y_batch in flow(image_list[-test_size:]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
score = model.test_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('test loss', score)])
return model, model.get_weights()
class LSTM_RNN:
def __init__(self, look_back, dropout_probability = 0.2, init ='he_uniform', loss='mse', optimizer='rmsprop'):
self.rnn = Sequential()
self.look_back = look_back
self.rnn.add(LSTM(10, stateful = True, batch_input_shape=(1, 1, 1), init=init))
self.rnn.add(Dropout(dropout_probability))
self.rnn.add(Dense(1, init=init))
self.rnn.compile(loss=loss, optimizer=optimizer)
def batch_train_test(self, trainX, trainY, testX, testY, nb_epoch=150):
print('Training LSTM-RNN...')
for epoch in range(nb_epoch):
print('Epoch '+ str(epoch+1) +'/{}'.format(nb_epoch))
training_losses = []
testing_losses = []
for i in range(len(trainX)):
y_actual = trainY[i]
for j in range(self.look_back):
training_loss = self.rnn.train_on_batch(np.expand_dims(np.expand_dims(trainX[i][j], axis=1), axis=1),
np.array([y_actual]))
training_losses.append(training_loss)
self.rnn.reset_states()
print('Mean training loss = {}'.format(np.mean(training_losses)))
mean_testing_loss = []
for i in range(len(testX)):
for j in range(self.look_back):
testing_loss = self.rnn.test_on_batch(np.expand_dims(np.expand_dims(testX[i][j], axis=1), axis=1),
np.array([testY[i]]))
testing_losses.append(testing_loss)
self.rnn.reset_states()
for j in range(self.look_back):
y_pred = self.rnn.predict_on_batch(np.expand_dims(np.expand_dims(testX[i][j], axis=1), axis=1))
self.rnn.reset_states()
mean_testing_loss = np.mean(testing_losses)
print('Mean testing loss = {}'.format(mean_testing_loss))
return mean_testing_loss
print('Iteration', iteration)
print("Training")
progbar = generic_utils.Progbar(train_num_samples)
gen = samples_generator(train_sequences, batch_size,
num_samples=train_num_samples)
for X, y in gen:
loss, accuracy = model.train_on_batch(X, y, accuracy=True)
progbar.add(batch_size, values=[("train loss", loss),
("train acc", accuracy)])
print()
print("Validating")
progbar = generic_utils.Progbar(valid_num_samples)
gen = samples_generator(valid_sequences, batch_size,
num_samples=valid_num_samples)
valid_loss = 0
for X, y in gen:
loss, accuracy = model.test_on_batch(X, y, accuracy=True)
progbar.add(batch_size, values=[("valid loss", loss),
("valid acc", accuracy)])
valid_loss += loss
print()
valid_loss /= float(valid_num_samples)
print("Valid Loss: {}, Best Loss: {}".format(valid_loss, best_loss))
if valid_loss < best_loss:
print("Saving model")
save_model(model, "sentence_model")
best_loss = valid_loss
def main():
# the data, shuffled and split between tran and test sets
data, labels = load_data(filename)
mat = scipy.io.loadmat(subjectsFilename, mat_dtype=True)
subjNumbers = np.squeeze(mat['subjectNum']) # subject IDs for each trial
# Creating the folds
# kf = StratifiedKFold(np.squeeze(labels), n_folds=ksplit, shuffle=True, random_state=123)
# kf = KFold(labels.shape[0], n_folds=ksplit, shuffle=True, random_state=123)
# fold_pairs = [(tr, ts) for (tr, ts) in kf]
# Leave-Subject-Out cross validation
fold_pairs = []
for i in np.unique(subjNumbers):
ts = subjNumbers == i
tr = np.squeeze(np.nonzero(np.bitwise_not(ts)))
ts = np.squeeze(np.nonzero(ts))
np.random.shuffle(tr) # Shuffle indices
np.random.shuffle(ts)
fold_pairs.append((tr, ts))
validScores, testScores = [], []
for foldNum, fold in enumerate(fold_pairs):
(X_train, y_train), (X_valid, y_valid), (X_test, y_test) = reformatInput(data, labels, fold)
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_valid.shape[0], 'valid samples')
print(X_test.shape[0], 'test samples')
X_train = X_train.astype("float32", casting='unsafe')
X_valid = X_valid.astype("float32", casting='unsafe')
X_test = X_test.astype("float32", casting='unsafe')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_valid = np_utils.to_categorical(y_valid, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
# Building the network
model = Sequential()
model.add(Convolution2D(40, 3, 3, border_mode='full',
input_shape=(image_dimensions, shapex, shapey)))
model.add(Activation('relu'))
model.add(Convolution2D(40, 3, 3))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
# model.add(Convolution2D(80, 3, 3, border_mode='full'))
# model.add(Activation('relu'))
# model.add(Convolution2D(80, 3, 3))
# model.add(Activation('relu'))
# model.add(MaxPooling2D(pool_size=(2, 2)))
# model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(1024))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# model.add(Convolution2D(nb_filters[0], image_dimensions, nb_conv[0], nb_conv[0], border_mode='full'))
# # model.add(BatchNormalization([nb_filters[0], nb_conv[0], nb_conv[0], image_dimensions]))
# model.add(Activation('relu'))
# model.add(Convolution2D(nb_filters[0], nb_filters[0], nb_conv[0], nb_conv[0]))
# model.add(Activation('relu'))
# model.add(MaxPooling2D(poolsize=(nb_pool[0], nb_pool[0])))
# model.add(Dropout(0.25))
#
# model.add(Convolution2D(nb_filters[1], nb_filters[0], nb_conv[0], nb_conv[0], border_mode='full'))
# model.add(Activation('relu'))
# model.add(Convolution2D(nb_filters[1], nb_filters[1], nb_conv[1], nb_conv[1]))
# model.add(Activation('relu'))
# model.add(MaxPooling2D(poolsize=(nb_pool[1], nb_pool[1])))
# model.add(Dropout(0.25))
#
# model.add(Flatten())
# # the image dimensions are the original dimensions divided by any pooling
# # each pixel has a number of filters, determined by the last Convolution2D layer
# model.add(Dense(nb_filters[-1] * (shapex / nb_pool[0] / nb_pool[1]) * (shapey / nb_pool[0] / nb_pool[1]), 1024))
# # model.add(BatchNormalization([1024]))
# model.add(Activation('relu'))
# model.add(Dropout(0.5))
# model.add(Dense(1024, nb_classes))
# model.add(Activation('softmax'))
# let's train the model using SGD + momentum (how original).
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
if not data_augmentation:
print("Not using data augmentation or normalization")
X_train = X_train.astype("float32", casting='unsafe')
X_test = X_test.astype("float32", casting='unsafe')
X_train /= 255.
X_test /= 255.
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True)
score, accu = model.evaluate(X_test, Y_test, batch_size=batch_size, show_accuracy=True)
print('Test accuracy:', accu)
else:
print("Using real time data augmentation")
# X_train = (X_train - np.mean(X_train, axis=0)) / np.std(X_train.flatten(), axis=0)
# X_valid = (X_valid - np.mean(X_valid, axis=0)) / np.std(X_valid.flatten(), axis=0)
# X_test = (X_test - np.mean(X_test, axis=0)) / np.std(X_test.flatten(), axis=0)
# X_train = (X_train - np.mean(X_train, axis=0))
# X_valid = (X_valid - np.mean(X_train, axis=0))
# X_test = (X_test - np.mean(X_train, axis=0))
# this will do preprocessing and realtime data augmentation
datagen = ImageDataGenerator(
featurewise_center=True, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0, # randomly shift images horizontally (fraction of total width)
height_shift_range=0, # randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
best_validation_accu = 0
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size=batch_size):
score, trainAccu = model.train_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("train accuracy", trainAccu)])
print("Validating...")
# Validation time!
progbar = generic_utils.Progbar(X_valid.shape[0])
epochValidAccu = []
for X_batch, Y_batch in datagen.flow(X_valid, Y_valid, batch_size=batch_size):
score, validAccu = model.test_on_batch(X_batch, Y_batch, accuracy=True)
epochValidAccu.append(validAccu)
progbar.add(X_batch.shape[0], values=[("validation accuracy", validAccu)])
meanValidAccu = np.mean(epochValidAccu)
if meanValidAccu > best_validation_accu:
best_validation_accu = meanValidAccu
best_iter = e
print("Testing...")
# test time!
progbar = generic_utils.Progbar(X_test.shape[0])
epochTestAccu = []
for X_batch, Y_batch in datagen.flow(X_test, Y_test, batch_size=batch_size):
score, testAccu = model.test_on_batch(X_batch, Y_batch, accuracy=True)
epochTestAccu.append(testAccu)
progbar.add(X_batch.shape[0], values=[("test accuracy", testAccu)])
model.save_weights('weigths_{0}'.format(foldNum), overwrite=True)
validScores.append(best_validation_accu)
testScores.append(np.mean(epochTestAccu))
scipy.io.savemat('cnn_results', {'validAccu': validScores, 'testAccu': testScores})
print ('Average valid accuracies: {0}'.format(np.mean(validScores)))
print ('Average test accuracies: {0}'.format(np.mean(testScores)))
for e in range(epoch_num):
print 'Epoch #{}/{}'.format(e+1,epoch_num)
sys.stdout.flush()
shuffle(tr_it)
for u in tqdm(tr_it):
l,a=model.train_on_batch(tr_in[u],tr_out[u])
tr_hist.r.addLA(l,a,tr_out[u].shape[0])
# clear_output()
tr_hist.log()
for u in range(dev_in.shape[0]):
l,a=model.test_on_batch(dev_in[u],dev_out[u])
dev_hist.r.addLA(l,a,dev_out[u].shape[0])
dev_hist.log()
# for u in range(tst_in.shape[0]):
# l,a=model.test_on_batch(tst_in[u],tst_out[u])
# tst_hist.r.addLA(l,a,tst_out[u].shape[0])
# tst_hist.log()
pickle.dump(model, open('models/classifier_enc.pkl','wb'))
pickle.dump(dev_hist, open('models/testHist_enc.pkl','wb'))
pickle.dump(tr_hist, open('models/trainHist_enc.pkl','wb'))
class LSTMOneHotLabels(AClassifier):
def __init__(self,
model_name,
tf_graph=None,
input_size=300,
dropout_rate=[0.2, 0.2],
layer_sizes=[100, 100],
activation=['tanh', 'tanh'],
optimizer='adam',
loss='mean_squared_error'):
self.inputSize = input_size
self.model_name = model_name
self.file_extension = '.h5'
self.tf_graph = tf_graph
if self.tf_graph is None:
self.tf_graph = KBackend.get_session().graph
with self.tf_graph.as_default():
self.model = Sequential()
self.model.add(
GaussianNoise(0.1, input_shape=(None, self.inputSize)))
for idx in range(len(layer_sizes) - 1):
self.model.add(
LSTM(layer_sizes[idx],
input_shape=(None, self.inputSize),
return_sequences=True,
batch_size=1,
activation=activation[idx]))
self.model.add(Dropout(dropout_rate[idx]))
self.model.add(
LSTM(layer_sizes[-1],
input_shape=(None, self.inputSize),
return_sequences=False,
batch_size=1,
activation=activation[-1]))
self.model.add(Dropout(dropout_rate[-1]))
self.model.add(Dense(10, activation='softmax'))
self.model.compile(optimizer=optimizer,
loss=loss,
metrics=['accuracy'])
# print(self.model.summary())
def train(self, X, Y, epochs=25, split=0.10, print_information=False):
splitVal = int(len(X) * split)
X_test = X[:splitVal]
Y_test = Y[:splitVal]
X_train = X[splitVal:]
Y_train = Y[splitVal:]
train_losses = []
train_acc = []
test_losses = []
test_acc = []
actual_acc = 0.00
try:
with self.tf_graph.as_default():
for idx in range(epochs):
if actual_acc < 0.60:
if print_information:
print("Epoch {}".format(idx))
epoch_train_loss = 0
epoch_train_acc = 0
epoch_test_loss = 0
epoch_test_acc = 0
for i in range(len(X_train)):
if print_information:
print("Training with text {}".format(i))
x_train = np.array(X_train[i])
y_train = np.array(Y_train[i])
x_train = x_train.reshape(1, len(x_train),
self.inputSize)
y_train = y_train.reshape(1, 10)
loss, acc = self.model.train_on_batch(
x_train, y_train)
epoch_train_loss += loss
epoch_train_acc += acc
for i in range(len(X_test)):
x_test = np.array(X_test[i])
y_test = np.array(Y_test[i])
x_test = x_test.reshape(1, len(x_test),
self.inputSize)
y_test = y_test.reshape(1, 10)
loss, acc = self.model.test_on_batch(
x_test, y_test)
epoch_test_loss += loss
epoch_test_acc += acc
train_losses.append(epoch_train_loss / len(X_train))
train_acc.append(epoch_train_acc / len(X_train))
test_losses.append(epoch_test_loss / len(X_test))
test_acc.append(epoch_test_acc / len(X_test))
if idx == epochs - 1 and print_information:
print('----------------')
print(
'epoch #{}\ntrain: loss: {:f} acc: {:f}\ntest: loss: {:f} acc: {:f}'
.format(idx, train_losses[-1], train_acc[-1],
test_losses[-1], test_acc[-1]))
actual_acc = test_acc[-1]
#print("====================")
#print("Epoch {}: \nloss_train = {} \nacc_train = {} \nloss_test = {} \nacc_test = {}".format(idx,loss_train,acc_train,loss_test,acc_test))
#print("====================")
else:
break
prediction_correct = 0
test_samples = 0
# calculate accuracy (test method in MODEL)
for i in range(0, len(X_test)):
if print_information:
print("Testing Batch - ", i)
x_test = np.array(X_test[i])
y_test = np.array(Y_test[i])
x_test = x_test.reshape(1, len(x_test), self.inputSize)
y_test = y_test.reshape(1, 10)
loss_test, acc_test = self.model.test_on_batch(
x_test, y_test)
test_samples += 1
if acc_test == 1.0:
prediction_correct += 1
accuracy = prediction_correct / test_samples
accuracy = accuracy * 100
if print_information:
print("feeding input --> FINISHED")
print("ACCURACY: ", accuracy, "%")
except BaseException as e:
print("[CLASSIFICATION] Error occurred during training: " + str(e))
raise RuntimeError("Error occurred during training")
if print_information:
pyplot.plot(train_losses, 'r-', test_losses, 'b-')
pyplot.xlabel('epoch')
pyplot.ylabel('loss')
pyplot.title(self.model_name + ' ' + actual_acc)
pyplot.savefig('data/training_stats/' + self.model_name +
'losses.png')
pyplot.close()
pyplot.plot(train_acc, 'r-', test_acc, 'b-')
pyplot.xlabel('epoch')
pyplot.ylabel('acc')
pyplot.title(self.model_name + ' ' + actual_acc)
pyplot.savefig('data/training_stats/' + self.model_name +
'acc.png')
pyplot.close()
def predict(self, X):
predictions = []
with self.tf_graph.as_default():
for i in range(len(X)):
x_predict = np.array(X[i])
x_predict = x_predict.reshape(1, len(x_predict),
self.inputSize)
predictions.append(self.model.predict_on_batch(x_predict))
return np.reshape(np.array(predictions),
(len(predictions), predictions[0].shape[1]))
def save(self, path):
path += self.file_extension
with self.tf_graph.as_default():
try:
self.model.save(path)
except BaseException as e:
print('Saving model to {} failed'.format(path))
raise SaveModelException(
'Saving model to {} failed'.format(path)) from e
def load(self, path):
path += self.file_extension
if not os.path.isfile(path):
raise LoadModelException(
"Loading model failed. File {} not found.".format(path))
with self.tf_graph.as_default():
try:
self.model = keras.models.load_model(path)
except OSError as e:
print('Loading model from {} failed'.format(path))
raise LoadModelException(
'Loading model from {} failed'.format(path)) from e
print('Iteration', iteration)
print("Training")
progbar = generic_utils.Progbar(train_num_samples)
gen = samples_generator(train_sentences, train_next_char, char_indices,
batch_size, num_samples=train_num_samples, X=X, y=y)
for X, y in gen:
loss, accuracy = model.train_on_batch(X, y, accuracy=True)
progbar.add(batch_size, values=[("train loss", loss), ("train acc",
accuracy)])
print("Validating")
progbar = generic_utils.Progbar(valid_num_samples)
gen = samples_generator(valid_sentences, valid_next_char, char_indices,
batch_size, num_samples=valid_num_samples, X=X, y=y)
valid_loss = 0
for X, y in gen:
loss, accuracy = model.test_on_batch(X, y, accuracy=True)
progbar.add(batch_size, values=[("valid loss", loss), ("valid acc",
accuracy)])
valid_loss += loss
valid_loss /= float(valid_num_samples)
print("Valid Loss: {}, Best Loss: {}".format(valid_loss, best_loss))
if valid_loss < best_loss:
print("Saving model")
save_model(model, "language_model")
best_loss = valid_loss
sample_model(model, char_indices, indices_char, random.choice(sentences))
window_size=4, negative_samples=test_labels, sampling_table=None)
tag_couples, labels = sequence.tagged_cbows(seq, test_tag_seq, len(tag_tokenizer.word_index)+1, # replace seq with train_tag_seq
window_size=4, negative_samples=test_labels, sampling_table=None)
couples = np.array(couples, dtype="int32")
tag_couples = np.array(tag_couples, dtype="int32")
labels = np.array(labels)
X_w = np.array(np.split(couples, len(seq)))
X_t = np.array(np.split(tag_couples, len(seq)))
if test_labels ==0 :
# Divide number of examples to rank so that GPU does not cause out of memory error
splitter = get_min_divisor(len(labels))
test_y = np.reshape(np.empty_like(labels, dtype = 'float32'),(labels.shape[0],1))
for j in range(splitter):
test_loss, test_y_block = model.test_on_batch([X_w[:,j*(labels.shape[0]/splitter): (j+1)*(labels.shape[0]/splitter) ,:],
X_t[:,j*(labels.shape[0]/splitter): (j+1)*(labels.shape[0]/splitter) ,:]],
labels[j*(labels.shape[0]/splitter): (j+1)*(labels.shape[0]/splitter)])
test_y[j*(labels.shape[0]/splitter): (j+1)*(labels.shape[0]/splitter)] = test_y_block
else:
test_loss, test_y = model.test_on_batch([X_w, X_t], labels)
lraps.append(label_ranking_average_precision_score(np.reshape(np.array(labels),test_y.shape).T , test_y.T))
mrr, recall, prec = test_utils.print_accuracy_results(np.array(labels) , np.reshape(test_y, np.array(labels).shape))
mrrs.append(mrr)
recalls.append(recall)
precs.append(prec)
losses.append(test_loss)
test_losses.append(test_loss)
if len(losses) % 100 == 0:
progbar.update(i, values=[("loss", np.sum(losses))])
losses = []
# Pooling layer1
model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
# Conv layer2 output shape (64,14,14)
model.add(Conv2D(64, 5, padding='same'))
model.add(Activation('relu'))
# Pooling layer2 (max pool) output shape (64,7,7)
model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
# FC
model.add(Flatten())
model.add(Dense(1024, activation='relu'))
model.add(Dense(10, activation='softmax'))
adam = Adam(epsilon=1e-8)
model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['acc'])
# start to fit model
print('Training------------------------')
model.fit(X_train, y_train, batch_size=32, epochs=2)
# test model
print('\nTesting---------------------------')
loss, accuracy = model.test_on_batch(X_test, y_test)
print('test loss:', loss)
print('test accuracy:', accuracy)
class rnnmlp ():
def __init__(self, r=(21, 109), dt=0.3):
self.r=r
self.dt=dt
self.rnnModel = Sequential()
self.maxFeatures=r[1]-r[0] +1
'''
simple RNN model,
'''
def SimpleRNNModel(self, nHidden=120, lr = 0.01):
self.rnnModel.add(SimpleRNN( nHidden, input_shape =( None, self.maxFeatures), activation='sigmoid', return_sequences=True))
self.rnnModel.add(TimeDistributedDense(self.maxFeatures))
self.rnnModel.add(Activation('softmax'))
rmsprop = RMSprop(lr=lr, rho=0.9, epsilon=1e-06)
self.rnnModel.compile(loss='categorical_crossentropy', optimizer=rmsprop)
'''
LSTM model
'''
def LSTMModel(self, nHidden=150, lr = 0.01):
# print('nHidden: %i\tlr: %.3f' % ( nHidden, lr) )
self.rnnModel.add(GRU( nHidden, activation='sigmoid', input_shape =( None, self.maxFeatures), return_sequences=True))
# self.rnnModel.add(LSTM( nHidden, activation='sigmoid', input_shape =( None, nHidden), return_sequences=True))
self.rnnModel.add(TimeDistributedDense(nHidden))
self.rnnModel.add(Activation('relu'))
self.rnnModel.add(TimeDistributedDense(self.maxFeatures))
self.rnnModel.add(Activation('softmax'))
rmsprop = RMSprop(lr=lr, rho=0.9, epsilon=1e-06)
self.rnnModel.compile(loss='categorical_crossentropy', optimizer=rmsprop)
'''
train module :
train model ,
file_name, the name of train or test file
weight_save_file, save the model parameters
'''
def train(self, file_name, weight_save_file, batch_size=1, num_epoch=200):
print('load data ---------------')
file_train=os.path.join(os.path.split(os.path.dirname(__file__))[0],
'data',file_name,'train','*.mid')
dataset = [midiread(f, self.r, self.dt).piano_roll.astype(theano.config.floatX) for f in glob.glob(file_train)]
file_test=os.path.join(os.path.split(os.path.dirname(__file__))[0],
'data',file_name,'test','*.mid')
testdataset = [midiread(f, self.r, self.dt).piano_roll.astype(theano.config.floatX) for f in glob.glob(file_test)]
print('load done --------------')
try:
for epoch in range(num_epoch):
t0 = time.time()
numpy.random.shuffle(dataset)
costs = []
accuracys = []
for s, sequence in enumerate(dataset):
y = numpy.hstack((sequence,numpy.zeros((sequence.shape[0],1)) ))
x = numpy.roll(y, 1, axis=0)
x[0,:]=0
x[0,self.maxFeatures-1]=1
cost, accuracy= self.rnnModel.train_on_batch(numpy.array([x]), numpy.array([y]), accuracy=True)
costs.append(cost)
accuracys.append(accuracy)
print('epoch: %i/%i\tcost: %.5f\taccu: %.5f\ttime: %.4f s' % (epoch+1, num_epoch, numpy.mean(costs), numpy.mean(accuracys),time.time()-t0))
sys.stdout.flush()
test_accu=self.evaluate(testdataset)
print('test_accu: %.5f' % ( numpy.mean(test_accu)) )
self.rnnModel.save_weights(weight_save_file)
except KeyboardInterrupt:
print('interrupt by user !')
'''
evaluate module :
evaluate model with test data, compute cost and accuracy
'''
def evaluate(self, test_dataset):
test_accuracy =[]
for s, sequence in enumerate(test_dataset):
test_y = numpy.hstack((sequence,numpy.zeros((sequence.shape[0],1)) ))
test_x = numpy.roll(test_y, 1, axis=0)
test_x[0,:]=0
test_x[0,self.maxFeatures-1]=1
cost, accu = self.rnnModel.test_on_batch(numpy.array([test_x]),numpy.array([test_y]), accuracy=True)
test_accuracy.append(accu)
return test_accuracy
'''
generate function :
generate music or chord,
init_chord: the first note of the generate sequence
file_name: file to save the sequence of generate notes
LS : if true , add Lsystem , generate chord
if false, no Lsystem, generate music notes
chord_name: chord name under condition of LS = True
chord_file: notes of all kinds of chords, load file
state_file: Lsystem model parameters, load file
n_steps: the length of generate sequence
r: notes which counts
'''
def generate(self, init_chord, file_name, LS=False, chord_name=None, chord_file=None, state_file=None, n_steps=80, r=(21,109)):
if(LS):
Lsystem = LSystem(chord_name, init_chord, chord_file, state_file, r)
init_sequence = numpy.zeros((1, n_steps +1, self.maxFeatures))
init_sequence[:, 0, init_chord-self.r[0]] = 1
for i in numpy.arange(n_steps):
probs = self.rnnModel.predict_proba(init_sequence)[:, i, :]
for j in numpy.arange(len(init_sequence)):
if(LS):
ind = Lsystem.getMaxProbs(probs[j,0:(self.maxFeatures-1)],True)
else:
ind = maxProbs(probs[j,0:(self.maxFeatures-1)])
init_sequence[j, i+1, ind] = 1
generate_sq = [sq[:,0:(self.maxFeatures-1)].nonzero()[1] for sq in init_sequence]
print(generate_sq[0] + self.r[0])
if(LS):
print(Lsystem.cur_chord)
print(Lsystem.cur_state)
print(Lsystem.cur_opes)
midiwrite(file_name, init_sequence[0,:,0:(self.maxFeatures-1)], self.r, self.dt)
extent = (0, self.dt * len(init_sequence[0,:,0:(self.maxFeatures-1)])) + self.r
pylab.figure()
pylab.imshow(init_sequence[0,:,0:(self.maxFeatures-1)].T, origin='lower', aspect='auto',interpolation='nearest', cmap=pylab.cm.gray_r,extent=extent)
pylab.xlabel('time (s)')
pylab.ylabel('MIDI note number')
pylab.title('generated piano-roll')
'''
load module: load model
'''
def loadModel(self, weight_save_file):
self.rnnModel.load_weights(weight_save_file)
def analyze_whale_data(**kwargs):
print("analyzing whale data")
# seed random
np.random.seed(kwargs["seed"])
# model inputs and parameters
#X_train = kwargs["X_train"]
#X_test = kwargs["X_test"]
#Y_train = kwargs["Y_train"]
#Y_test = kwargs["Y_test"]
X_train=np.load(home+'/gabor/numpyFiles/Training Set.npy')
X_test=np.load(home+'/gabor/numpyFiles/TestSet.npy')
Y_train=np.load(home+'/gabor/numpyFiles/Training Labels.npy')
Y_test=np.load(home+'/gabor/numpyFiles/TestSet Labels.npy')
X_test = X_test.reshape(-1, 1, 30, 96)
Y_test = np_utils.to_categorical(Y_test, 447)
X_train = X_train.reshape(-1, 1, 30, 96)
Y_train = np_utils.to_categorical(Y_train, 447)
img_rows = kwargs["img_rows"]
img_cols = kwargs["img_cols"]
nb_filters = kwargs.get("nb_filters", 32)
nb_conv = kwargs.get("nb_conv", 3)
nb_pool = kwargs.get("nb_pool", 2)
batch_size = kwargs.get("batch_size", 32)
nb_epoch = kwargs.get("nb_epoch", 12)
nb_classes = kwargs.get("nb_classes", 447)
print("X_test.shape == {};".format(X_test.shape))
print("Y_test.shape == {};".format(Y_test.shape))
print("X_test.shape == {};".format(X_train.shape))
print("Y_test.shape == {};".format(Y_train.shape))
# CNN architecture
print("--> creating CNN network ...")
model = Sequential()
# layer 1
model.add(
Convolution2D(
nb_filters,
nb_conv,
nb_conv,
border_mode='full',
input_shape=(1, img_rows, img_cols)
)
)
model.add(Activation('relu'))
# layer 2
model.add(Convolution2D(32, 5, 5, border_mode='valid'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
# layer 3
model.add(Convolution2D(64, 5, 5, border_mode='valid'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
# layer 4
model.add(Convolution2D(128, 3, 3, border_mode='valid'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
# layer 5
model.add(Flatten())
model.add(Dense(1000))
model.add(Activation('relu'))
model.add(Dropout(0.5))
# layer 6
model.add(Dense(1000))
model.add(Activation('relu'))
model.add(Dropout(0.5))
# layer 4
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# compile, fit and evaluate model
print("--> compiling CNN functions")
model.compile(
loss='categorical_crossentropy',
optimizer='adadelta'
)
print("--> fitting CNN")
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=0, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.1, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.1, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=True) # randomly flip images
datagen.fit(X_train)
loss1 = np.zeros((nb_epoch))
loss1 = loss1.astype(np.float32)
acc1 = np.zeros((nb_epoch))
acc1 = acc1.astype(np.float32)
score1 = np.zeros((nb_epoch))
score1 = score1.astype(np.float32)
test_acc1 = np.zeros((nb_epoch))
test_acc1 = test_acc1.astype(np.float32)
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train):
loss, acc = model.train_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("train loss", loss), ("train accuracy", acc)])
loss1[e] = loss
acc1[e] = acc
print("Testing...")
# test time!
progbar = generic_utils.Progbar(X_test.shape[0])
for X_batch, Y_batch in datagen.flow(X_test, Y_test):
score, test_acc = model.test_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("test loss", score), ("test accuracy", test_acc)])
score1[e] = score
test_acc1[e] = test_acc
print("--> saving CNN")
json_string = model.to_json()
open('my_model_architecture.json', 'w').write(json_string)
model.save_weights('my_model_weights.h5')
return (loss1, acc1, score1, test_acc1)
X, y = get_data(files[n:n+batch_size], n)
else:
X, y = get_data(files[n:], n)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
X_train = np.array(X_train)
X_test = np.array(X_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model.train_on_batch(X_train, Y_train)
l, a = model.test_on_batch(X_test, Y_test, accuracy=True)
acc.append(a)
loss.append(l)
print ''
print "Val_loss", (sum(loss) / len(loss))
print "Val_acc", (sum(acc) / len(acc))
# with random batch draws....
# -------
# Epoch 1
# Val_loss 0.889370705837
# Val_acc 0.479363625794
# -------
# Epoch 2
def train(shared_variables):
shared_variables["modelEvent"].wait()
shared_variables["modelEvent"].clear()
global env
global roads
for key in shared_variables["lanes"]:
shared_variables["lanes"][key] = Lane(key)
# print(shared_variables["lanes"])
# sys.exit()
print(shared_variables["edges_lanes"])
roads = [
Road("zeytouna-ain_mreysse", 100, 100, [
shared_variables["lanes"]["zeytouna-ain_mreysse_0"],
shared_variables["lanes"]["zeytouna-ain_mreysse_1"],
shared_variables["lanes"]["zeytouna-ain_mreysse_2"],
shared_variables["lanes"]["zeytouna-ain_mreysse_3"]
]),
Road("aub-ain_mreysse", 100, 100, [
shared_variables["lanes"]["aub-ain_mreysse_0"],
shared_variables["lanes"]["aub-ain_mreysse_1"]
]),
Road("bliss-ain_mreysse", 100, 200, [
shared_variables["lanes"]["bliss-ain_mreysse_0"],
shared_variables["lanes"]["bliss-ain_mreysse_1"]
])
]
# for edge, lanes in shared_variables["edges_lanes"].items():
# lanes_input = []
# for lane in lanes:
# lanes_input.append(shared_variables["lanes"][lane])
# roads.append(Road(edge,shared_variables["max_queue_lengths"][edge],shared_variables["max_time_delays"][edge],lanes_input))
# Define environment/game
num_phases = shared_variables["num_phases"]
env = Simulation(shared_variables["current_time"], roads, num_phases)
phase_duration = 0
num_lanes = env.get_total_num_lanes()
input_size = 1 + 2 * num_lanes
# Define Model
model = Sequential()
model.add(Dense(input_size, input_shape=(input_size, ), activation='relu'))
model.add(Dense(input_size, activation='relu'))
model.add(Dense(num_phases))
model.compile(sgd(lr=shared_variables["learning_rate"]),
shared_variables["optimizer"])
try:
model.load_weights("model.h5")
except:
print("failed to load model")
# Initialize experience replay object
exp_replay = ExperienceReplay(shared_variables["max_memory"],
shared_variables["discount"])
shared_variables["lock"].acquire()
while not shared_variables["doneEvent"].is_set():
shared_variables["lock"].release()
shared_variables["modelEvent"].wait()
shared_variables["modelEvent"].clear()
state = "\nold state:\t\t" + str(
np.asarray(env.get_old_state()[0]).tolist())
state += "\nnew phase:\t\t" + str(env.get_current_phase())
state += "\nreward:\t\t\t" + str(env.get_reward())
state += "\nnew state:\t\t" + str(
np.asarray(env.get_state()[0]).tolist()) + "\n"
if shared_variables["mode"] == "train":
# # adapt model
exp_replay.remember([
env.get_old_state(),
env.get_current_phase(),
env.get_reward(),
env.get_state()
])
inputs, targets = exp_replay.get_batch(
model, batch_size=shared_variables["batch_size"])
shared_variables["loss"] += model.train_on_batch(inputs, targets)
elif shared_variables["mode"] == "test_model":
exp_replay.remember([
env.get_old_state(),
env.get_current_phase(),
env.get_reward(),
env.get_state()
])
inputs, targets = exp_replay.get_batch(
model, batch_size=shared_variables["batch_size"])
shared_variables["loss"] += model.test_on_batch(inputs, targets)
for edge, lanes in shared_variables["edges_lanes"].items():
shared_variables["cumulative_queue_length_" + edge] += \
[road for road in env.get_roads() if road.name == edge][0].get_avg_queue_lengths()
shared_variables["cumulative_time_delay_" + edge] += \
[road for road in env.get_roads() if road.name == edge][0].get_avg_time_delays()
phase_duration += 1
env.set_old_state()
# if phase_duration == shared_variables["phase_duration"]:
phase_duration = 0
# get next phase
if shared_variables["mode"] == "train" and np.random.rand(
) <= shared_variables["exploration"]:
new_phase = np.random.randint(0, num_phases, size=1)[0]
else:
q = model.predict(env.get_state())
new_phase = np.argmax(q[0])
env.set_current_phase(new_phase)
shared_variables["new_phase"] = env.get_current_phase(
) # if not shared_variables["accumulation"] or accumulation_counter > shared_variables["accumulate_duration"] else shared_variables["accumulate_phase"]
# apply new phase, get rewards and new state
# state += "\nnew phase:\t\t" + str(shared_variables["new_phase"]) + "\n"
# print(state)
shared_variables["states"] += state
shared_variables["lock"].acquire()
shared_variables["serverEvent"].set()
if shared_variables["mode"] == "train":
model.save_weights("model.h5", overwrite=True)
# with open("model.json", "w") as outfile:
# json.dump(model.to_json(), outfile)
shared_variables["lock"].release()
print("Model exiting")
def train_whale_data(**kwargs):
print("analyzing whale data")
# seed random
np.random.seed(kwargs["seed"])
# model inputs and parameters
data_augmentation = kwargs.get("data_augmentation", False)
X_train = kwargs["X_train"]
X_test = kwargs["X_test"]
Y_train = kwargs["Y_train"]
Y_test = kwargs["Y_test"]
img_rows = kwargs["img_rows"]
img_cols = kwargs["img_cols"]
nb_filters = kwargs.get("nb_filters", 32)
nb_conv = kwargs.get("nb_conv", 3)
nb_pool = kwargs.get("nb_pool", 2)
batch_size = kwargs["batch_size"]
nb_epoch = kwargs.get("nb_epoch", 12)
nb_classes = kwargs.get("nb_classes", 10)
model_file = kwargs["model_file"]
weights_file = kwargs["weights_file"]
results_file = kwargs["results_file"]
# CNN architecture
print("--> creating CNN network ...")
results = {
"acc": [],
"val_acc": [],
"loss": [],
"val_loss": []
}
model = Sequential()
# layer 1
model.add(
Convolution2D(
nb_filters,
nb_conv,
nb_conv,
input_shape=(1, img_rows, img_cols),
)
)
model.add(Activation('relu'))
model.add(Dropout(0.4))
# layer 2
model.add(
Convolution2D(
nb_filters,
nb_conv,
nb_conv,
input_shape=(1, img_rows, img_cols),
)
)
model.add(Activation('relu'))
model.add(Dropout(0.4))
# layer 3
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.4))
# layer 4
model.add(Flatten())
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
# compile, fit and evaluate model
print("--> compiling CNN functions")
model.compile(
loss='categorical_crossentropy',
optimizer='sgd'
)
# fit model
print("--> fitting CNN")
if data_augmentation is False:
print "--> fitting data"
fitlog = model.fit(
X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
verbose=1,
validation_data=(X_test, Y_test)
)
results = fitlog.history
else:
# turn on data augmentation
print "--> augmenting data"
datagen = ImageDataGenerator(
featurewise_center=False,
featurewise_std_normalization=False,
rotation_range=20,
width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
)
datagen.fit(X_train)
print "--> fitting data"
for e in range(nb_epoch):
print "epoch:", e
for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size):
train_loss, train_accuracy = model.train_on_batch(
X_batch,
Y_batch,
accuracy=True
)
valid_loss, valid_accuracy = model.test_on_batch(
X_test,
Y_test,
accuracy=True
)
results["acc"].append(float(train_accuracy))
results["val_acc"].append(float(valid_accuracy))
results["loss"].append(float(train_loss))
results["val_loss"].append(float(valid_loss))
print "acc: {0}".format(train_accuracy),
print "val_acc: {0}".format(valid_accuracy),
print "acc_loss: {0}".format(train_loss),
print "val_loss: {0}".format(valid_loss)
# save model
model_data = model.to_json()
model_file = open(model_file, "w")
model_file.write(json.dumps(model_data))
model_file.close()
# save model weights
model.save_weights(weights_file, overwrite=True)
# save results
results["nb_epoch"] = nb_epoch
results["batch_size"] = batch_size
rf = open(results_file, "w")
rf.write(json.dumps(results))
rf.close()
print('-'*40)
print('Epoch', e)
print('-'*40)
print('Training...')
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train,batch_size=128):
loss = modelCascade.train_on_batch(X_batch, Y_batch,accuracy=True)
progbar.add(X_batch.shape[0], values=[('train loss', loss[0]),('train accuracy',loss[1])])
print('Testing...')
# test time!
accuracyArray = list()
lossArray = list()
progbar = generic_utils.Progbar(X_test.shape[0])
for X_batch, Y_batch in datagen.flow(X_test, Y_test):
score = modelCascade.test_on_batch(X_batch, Y_batch,accuracy=True)
lossArray.append(score[0])
accuracyArray.append(score[1])
progbar.add(X_batch.shape[0], values=[('test loss', score[0]),('test accuracy',score[1])])
lossIteration1.append(np.mean(lossArray))
accuracyIteration1.append(np.mean(accuracyArray))
weightsLayer1 = modelCascade.layers[0].get_weights()
print('SECOND ITERATION STARTING')
#####SECOND ITERATION
#MODEL THAT IS USED TO GENERATE INPUT HAT
modelIH = Sequential()
modelIH.add(Convolution2D(32, 3, 3, border_mode='same',
input_shape=(img_channels, img_rows, img_cols),weights=weightsLayer1))
modelIH.add(Activation('relu'))
def cnn(**kwargs):
verbose = kwargs.get("verbose", True)
X_train = kwargs["X_train"]
Y_train = kwargs["Y_train"]
X_test = kwargs["X_test"]
Y_test = kwargs["Y_test"]
input_shape = kwargs["input_shape"]
nb_classes = kwargs["nb_classes"]
data_augmentation = kwargs.get("data_augmentation", True)
nb_convo_layers = kwargs["nb_convo_layers"]
nb_filters = kwargs["nb_filters"]
nb_conv = kwargs["nb_conv"]
convo_activations = kwargs["convo_activations"]
maxpools = kwargs["maxpools"]
pool_sizes = kwargs["pool_sizes"]
convo_dropouts = kwargs["convo_dropouts"]
nb_dense_layers = kwargs["nb_dense_layers"]
dense_hidden_neurons = kwargs["dense_hidden_neurons"]
dense_activations = kwargs["dense_activations"]
dense_dropouts = kwargs["dense_dropouts"]
loss = kwargs["loss"]
optimizer = kwargs["optimizer"]
nb_epoch = kwargs["nb_epoch"]
batch_size = kwargs["batch_size"]
model_file = kwargs.get("model_file")
weights_file = kwargs.get("weights_file")
results_file = kwargs.get("results_file")
results = {
"acc": [],
"loss": [],
"val_acc": [],
"val_loss": []
}
# CNN architecture
model = Sequential()
# convolution layers
for i in range(nb_convo_layers):
# convo layer
if i == 0:
model.add(
Convolution2D(
nb_filters[i],
nb_conv[i],
nb_conv[i],
input_shape=input_shape
)
)
else:
model.add(
Convolution2D(
nb_filters[i],
nb_conv[i],
nb_conv[i],
border_mode='valid',
)
)
# activation
if convo_activations[i]:
model.add(Activation(convo_activations[i]))
# max-pooling
if maxpools[i]:
model.add(MaxPooling2D(pool_size=(pool_sizes[i], pool_sizes[i])))
# dropout
if convo_dropouts[i]:
model.add(Dropout(convo_dropouts[i]))
# dense layers
model.add(Flatten())
for i in range(nb_dense_layers):
# dense layer
if (i + 1) == nb_dense_layers:
model.add(Dense(nb_classes))
else:
model.add(Dense(dense_hidden_neurons[i]))
# activation
if dense_activations[i]:
model.add(Activation(dense_activations[i]))
# dropout
if dense_dropouts[i]:
model.add(Dropout(dense_dropouts[i]))
# loss function and optimizer
if verbose:
print("--> compiling CNN")
model.compile(loss=loss, optimizer=optimizer)
# fit model
if verbose:
print("--> fitting CNN")
if data_augmentation is False:
fitlog = model.fit(
X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
show_accuracy=True,
verbose=verbose,
validation_data=(X_test, Y_test)
)
results = fitlog.history
else:
# turn on data augmentation
datagen = ImageDataGenerator(
featurewise_center=False,
samplewise_center=True,
featurewise_std_normalization=False,
samplewise_std_normalization=False,
zca_whitening=False,
rotation_range=0,
width_shift_range=0.1,
height_shift_range=0.1,
horizontal_flip=True,
vertical_flip=True
)
datagen.fit(X_train)
for e in range(nb_epoch):
if verbose:
print "epoch:", e
tmp_train_acc = []
tmp_train_loss = []
tmp_test_acc = []
tmp_test_loss = []
train_batch_counter = 0
test_batch_counter = 0
for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size):
train_loss, train_accuracy = model.train_on_batch(
X_batch,
Y_batch,
accuracy=True
)
tmp_train_acc.append(train_accuracy)
tmp_train_loss.append(train_loss)
train_batch_counter += 1
for X_batch, Y_batch in datagen.flow(X_test, Y_test, batch_size):
valid_loss, valid_accuracy = model.test_on_batch(
X_batch,
Y_batch,
accuracy=True
)
tmp_test_acc.append(valid_accuracy)
tmp_test_loss.append(valid_loss)
test_batch_counter += 1
epoch_train_acc = sum(tmp_train_acc) / float(train_batch_counter)
epoch_train_loss = sum(tmp_train_loss) / float(train_batch_counter)
epoch_test_acc = sum(tmp_test_acc) / float(test_batch_counter)
epoch_test_loss = sum(tmp_test_loss) / float(test_batch_counter)
results["acc"].append(epoch_train_acc)
results["loss"].append(epoch_train_loss)
results["val_acc"].append(epoch_test_acc)
results["val_loss"].append(epoch_test_loss)
if verbose:
print "acc: {0}".format(epoch_train_acc),
print "loss: {0}".format(epoch_train_loss),
print "val_acc: {0}".format(epoch_test_acc),
print "val_loss: {0}".format(epoch_test_loss)
# save model
if model_file:
model_data = model.to_json()
model_file = open(model_file, "w")
model_file.write(json.dumps(model_data))
model_file.close()
# save model weights
if weights_file:
model.save_weights(weights_file, overwrite=True)
# save results
if results_file:
results["nb_epoch"] = nb_epoch
results["batch_size"] = batch_size
rf = open(results_file, "w")
rf.write(json.dumps(results))
rf.close()
# evaluate
score = model.evaluate(
X_test,
Y_test,
show_accuracy=True,
verbose=verbose,
batch_size=batch_size
)
return results, score
]) / size_train * nepochs)
print(
'The same but with the predictions computed in a different way >>>',
sum([
round(auth) == y[0][0][0]
for auth, y in zip(pred_author_train2, Y_train * nepochs)
]) / size_train * nepochs)
print('Evaluate...')
# reinitialize metrics
loss, acc = 0, 0
test_metrics = np.empty((size_test, 2))
pred_author_test = []
pred_author_test2 = []
for i, (x_test, y_test) in enumerate(zip(X_test, Y_test)):
# considering all predictions for each sequence
test_metrics[i] = model.test_on_batch(x_test, y_test)
# consider only the last prediction for each sequence
pred_author_test.append(sigmoid1_out(lstm1_out([x_test]))[0][0, -1, 0])
pred_author_test2.append(model.predict(x_test)[0, -1, 0])
if (i + 1) % step_len == 0:
nsteps = (i + 1) // step_len
loss = inc_avg(loss,
np.mean(test_metrics[(nsteps - 1) * step_len:i + 1, 0]),
nsteps)
acc = inc_avg(acc,
np.mean(test_metrics[(nsteps - 1) * step_len:i + 1, 1]),
nsteps)
print('iteration {2: <5} > loss: {0} - acc: {1}'.format(
loss, acc, i + 1))
if (size_test % step_len
) != 0: # all batch means computed, update means with remaining metrics
tr_loss, tr_acc = model.train_on_batch(x_in, y_in)
mean_tr_acc.append(tr_acc)
mean_tr_loss.append(tr_loss)
# move this into the inner loop and watch the network never learn
# move into inner loop and reset every x < recall_len for fancy failure
model.reset_states()
print('accuracy training = {}'.format(np.mean(mean_tr_acc)))
print('loss training = {}'.format(np.mean(mean_tr_loss)))
print('___________________________________')
mean_te_acc = []
mean_te_loss = []
for seq_idx in range(X_test.shape[0]):
start_val = X_test[seq_idx, 0]
assert y_test[seq_idx] == start_val
assert tuple(np.nonzero(X_test[seq_idx, :]))[0].shape[0] == start_val
y_in = np.array([y_test[seq_idx]], dtype=np.bool)
for j in range(recall_len):
te_loss, te_acc = model.test_on_batch(np.array([[[X_test[seq_idx][j]]]], dtype=np.bool), y_in)
mean_te_acc.append(te_acc)
mean_te_loss.append(te_loss)
model.reset_states()
print('accuracy testing = {}'.format(np.mean(mean_te_acc)))
print('loss testing = {}'.format(np.mean(mean_te_loss)))
print('___________________________________')
callbacks = [early_stop, save_best]
# Train model
t0 = time.time()
hist = model.fit(train_set[0],
train_set[1],
validation_data=valid_set,
verbose=2,
callbacks=callbacks,
nb_epoch=1000,
batch_size=20)
time_elapsed = time.time() - t0
# Load best model
model.load_weights("models/%s.mdl" % MDL_NAME)
# Print time elapsed and loss on testing dataset
test_set_loss = model.test_on_batch(test_set[0], test_set[1])
print "\nTime elapsed: %f s" % time_elapsed
print "Testing set loss: %f" % test_set_loss
# Save results
qri.save_results("results/%s.out" % MDL_NAME, time_elapsed, test_set_loss)
qri.save_history("models/%s.hist" % MDL_NAME, hist.history)
# Plot training and validation loss
qri.plot_train_valid_loss(hist.history)
# Make predictions
qri.plot_test_predictions(model, train_set)
inputs /= 255
loss, acc = model.train_on_batch(inputs, targets)
train_loss += loss
train_acc += acc
train_batches += 1
# And a full pass over the validation data:
val_loss = 0
val_acc = 0
val_batches = 0
test_str.reset() # re-start streaming
for inputs, targets, pad in val_str:
if pad: # not full batch
break
inputs /= 255
loss, acc = model.test_on_batch(inputs, targets)
val_loss += loss
val_acc += acc
val_batches += 1
# Then we print the results for this epoch:
print("Epoch {}/{} time {:.3f}s; train loss: {:.6f} acc:{:.6f}; val loss: {:.6f} acc:{:.2f}".format(
epoch + 1, nb_epoch, time.time() - start_time,
train_loss/train_batches, train_acc/train_batches,
val_loss/val_batches, val_acc/val_batches))
# After training, we compute and print the test error:
test_loss = 0
test_acc = 0
test_batches = 0
for inputs, targets, pad in test_str:
if i % 10 == 0:
print('Batch : ', i, '/',
train_one_batch, ', loss in minibatch: ',
float(train_loss), ', acc in minibatch: ',
float(train_acc), 'current best: ', best_acc)
avg_train_loss += float(train_loss)
avg_train_acc += float(train_acc)
if i % 100 == 0:
avg_dev_acc = 0.0
dev_data.shuffle()
for j, (data_, sentiments_) in enumerate(
_batch_loader(dev_data, config.batch)):
_, dev_acc = model.test_on_batch(data_, sentiments_)
avg_dev_acc += float(dev_acc)
cur_acc = avg_dev_acc / dev_one_batch
print('Epoch : ', epoch, 'Batch : ', i, '/',
train_one_batch, 'Validation ACC : ', cur_acc)
if cur_acc >= best_acc and config.savemodel == True:
best_acc = cur_acc
print(
'################### Best Acc Found #############'
)
model.save('./modelsave/{}epoch'.format(epoch) +
config.savename)
print('Test score:', score)
else:
print('Using real time data augmentation')
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print('Training...')
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(num_images-test_size)
for X_batch, Y_batch in flow(image_list[0:-test_size]):
X_batch = X_batch.reshape(X_batch.shape[0], img_channels, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('train loss', loss)])
print('Testing...')
# test time!
progbar = generic_utils.Progbar(test_size)
for X_batch, Y_batch in flow(image_list[-test_size:]):
X_batch = X_batch.reshape(X_batch.shape[0], img_channels, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
score = model.test_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('test loss', score)])
json_string = model.to_json()
open('cnn1_model_architecture.json', 'w').write(json_string)
model.save_weights('cnn1_model_weights.h5')
'''
i=0
for file in fileList:
print file
print i
x_train,y_train=readPklFile(file)
#print x_train
if x_train.ndim<3:
continue
if(len(x_train)==0):
continue
if i<1800:
model.fit(x_train,y_train,nb_epoch=40,batch_size=32,verbose=1,show_accuracy=True)
else:
classes=model.predict(x_train,batch_size=32,verbose=1)
y_test=Nomalization(classes)
predictEntity.writelines(file+'\n'+"original:\n"+str(y_train)+"\npredict:\n"+str(y_test)+'\n')
loss,acc=model.test_on_batch(x_train,y_train,accuracy=True)
#print loss,acc
#acc= CalculateAcc(y_train,y_test)
FileEntity.writelines(file+'\n'+'loss: '+str(loss)+"\tacc: "+str(acc)+"\n")
#model.evaluate(x_train,y_train,batch_size=32,verbose=1,show_accuracy=True)
i=i+1
FileEntity.close()
predictEntity.close()
print "finish"
loss, accuracy = model.train_on_batch(b, l)
end_time = time.time()
print('batch {}/{} loss: {} accuracy: {} time: {}ms'.format(
int(current_index / batch_size), int(nice_n / batch_size), loss,
accuracy, 1000 * (end_time - start_time)),
flush=True)
print('epoch {}/{}'.format(i, epochs))
current_index = 0
loss = 0.0
acc = 0.0
while current_index + batch_size < len(training_images):
b, l = get_batch()
score = model.test_on_batch(b, l)
print('Test batch score:', score[0])
print('Test batch accuracy:', score[1], flush=True)
loss += score[0]
acc += score[1]
loss = loss / int(nice_n / batch_size)
acc = acc / int(nice_n / batch_size)
print('Test score:', loss)
print('Test accuracy:', acc)
horizontal_flip=False, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train):
score, acc = model.test_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("train accuracy", acc)])
print("Testing...")
progbar = generic_utils.Progbar(X_test.shape[0])
for X_batch, Y_batch in datagen.flow(X_test, Y_test):
score, acc = model.test_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("test accuracy", acc)])
# test time!
for X_batch, Y_batch in datagen.flow(X_te_orig, np.ones((1,X_te_orig.shape[0])), batch_size = X_te_orig.shape[0]):
y_te = model.predict_classes(X_batch)
save_out(y_te,labels_string,sorted_files_te,submission_fname)
# model.add(Activation("relu"))
# model.add(Dense(3, kernel_initializer="normal"))
# model.compile(optimizer=Adam(lr=1e-6), loss="mse")
tensorboard = TensorBoard(log_dir="logs/{}".format(time.time()))
print(x_train)
print(y_train)
history = model.fit(x_train,
y_train,
epochs=100,
batch_size=100,
verbose=1,
callbacks=[tensorboard],
shuffle=True)
model.test_on_batch(x_test, y_test)
scores = model.evaluate(x_test, y_test)
print(model.summary())
import os
os.environ["PATH"] += os.pathsep + 'C:/Program Files (x86)/Graphviz2.38/bin/'
plot_model(model, to_file="plot.png", show_layer_names=True, show_shapes=True)
print(history.history.keys())
# "Accuracy"
plt.plot(history.history['acc'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper left')
print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format(avgLoss, con_dict['r'], con_dict['p'], con_dict['f1']))
print("Validating =>")
val_pred_label = []
avgLoss = 0
bar = progressbar.ProgressBar(max_value=len(val_x))
for n_batch, sent in bar(enumerate(val_x)):
label = val_label[n_batch]
label = np.eye(n_classes)[label][np.newaxis,:]
sent = sent[np.newaxis,:]
if sent.shape[1] > 1: #some bug in keras
loss = model.test_on_batch(sent, label)
avgLoss += loss
pred = model.predict_on_batch(sent)
pred = np.argmax(pred,-1)[0]
val_pred_label.append(pred)
avgLoss = avgLoss/n_batch
predword_val = [ list(map(lambda x: idx2la[x], y)) for y in val_pred_label]
con_dict = conlleval(predword_val, groundtruth_val, words_val, 'r.txt')
val_f_scores.append(con_dict['f1'])
print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format(avgLoss, con_dict['r'], con_dict['p'], con_dict['f1']))
if con_dict['f1'] > best_val_f1:
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train):
loss = model.train_on_batch(X_batch, X_batch.reshape(X_batch.shape[0],X_train.shape[2]**2*3))
progbar.add(X_batch.shape[0], values=[("train loss", loss)])
print("Testing...")
# test time!
progbar = generic_utils.Progbar(X_test.shape[0])
for X_batch, Y_batch in datagen.flow(X_test, X_test):
score = model.test_on_batch(X_batch, X_batch.reshape(X_batch.shape[0],X_train.shape[2]**2*3))
progbar.add(X_batch.shape[0], values=[("test loss", score)])
model2 = Sequential()
model2.add(encoder)
codes = []
targets = []
model2.compile(loss = "mean_squared_error", optimizer = "sgd")
for X_batch, Y_batch in datagen.flow(X_train, Y_train):
codes.append(model2.predict(X_batch))
targets.append(np.argmax(Y_batch))
print('stack it...')
codes = np.vstack(codes)
targets = np.vstack(targets)
print(codes.shape,'code shape')
train_f_scores.append(con_dict['f1'])
print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format(
avgLoss, con_dict['r'], con_dict['p'], con_dict['f1']))
print("Validating =>")
val_pred_label = []
avgLoss = 0
bar = progressbar.ProgressBar(max_value=len(val_x))
for n_batch, sent in bar(enumerate(val_x)):
label = val_label[n_batch]
label = np.eye(n_classes)[label][np.newaxis, :]
sent = sent[np.newaxis, :]
if sent.shape[1] > 1:
loss = model.test_on_batch(sent, label)
avgLoss += loss
pred = model.predict_on_batch(sent)
pred = np.argmax(pred, -1)[0]
val_pred_label.append(pred)
avgLoss = avgLoss / n_batch
predword_val = [list(map(lambda x: idx2la[x], y)) for y in val_pred_label]
con_dict = conlleval(predword_val, groundtruth_val, words_val, 'r.txt')
val_f_scores.append(con_dict['f1'])
print('Loss = {}, Precision = {}, Recall = {}, F1 = {}'.format(
avgLoss, con_dict['r'], con_dict['p'], con_dict['f1']))
#TESTING GOES HERE
print('Testing...')
accuracyArray = list()
lossArray = list()
progbar = generic_utils.Progbar(X_test.shape[0])
j = 0
while(j != sizeTest):
if(j+batch_size >= sizeTest):
batchToTest = model2.predict(X_test[j:sizeTest])
labelOfBatchToTest = Y_test[j:sizeTest,:]
j = sizeTest
else:
batchToTest = model2.predict(X_test[j:j+batch_size])
labelOfBatchToTest = Y_test[j:j+batch_size,:]
j += batch_size
score = model3.test_on_batch(batchToTest, labelOfBatchToTest,accuracy=True)
lossArray.append(score[0])
accuracyArray.append(score[1])
progbar.add(batchToTest.shape[0], values=[('test loss', score[0]),('test accuracy',score[1])])
lossIteration2.append(np.mean(lossArray))
accuracyIteration2.append(np.mean(accuracyArray))
weightsLayer2 = model3.layers[0].get_weights()
print('ITERATION 2 STARTING (FW)')
#PREVIOUS MODEL
model2 = Sequential()
model2.add(Convolution2D(96, nb_conv, nb_conv,
input_shape=(1, img_rows, img_cols),weights=weightsLayer1))
model2.add(Activation('relu'))
class LanguageModel(RNNModel):
def __init__(self, *args, **kwargs):
'''
field is 'sentence' by default, but can be e.g. pos_sentence.
'''
self.field = kwargs.pop('field', 'sentence')
super(LanguageModel, self).__init__(*args, **kwargs)
self.class_to_code = {'VBZ': 0, 'VBP': 1}
self.inflect_verb, _ = gen_inflect_from_vocab(self.vocab_file)
def process_single_dependency(self, dep):
dep['label'] = dep['verb_pos']
tokens = dep[self.field].split()
return tokens
def create_train_and_test(self, examples):
random.seed(1)
random.shuffle(examples)
first = 1
self.X_train = []
self.Y_train = []
self.X_test = self.Y_test = [] # not used; just for compatibility
self.deps_train = []
n_train = int(len(examples) * self.prop_train)
for _, ints, dep in examples[:n_train]:
self.deps_train.append(dep)
for i in range(first, len(ints) - 1):
self.X_train.append(ints[:i])
self.Y_train.append(ints[i])
self.Y_train = np.asarray(self.Y_train)
self.deps_test = [x[2] for x in examples[n_train:]]
def create_model(self):
self.log('Creating model')
self.model = Sequential()
self.model.add(
Embedding(len(self.vocab_to_ints) + 1,
self.embedding_size,
input_length=self.maxlen))
self.model.add(
self.rnn_class(output_dim=self.rnn_output_size,
input_length=self.maxlen))
self.model.add(Dense(len(self.vocab_to_ints) + 1))
self.model.add(Activation('softmax'))
def compile_model(self):
self.log('Compiling model')
self.model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam')
def results(self):
recs = []
columns = ['gram_loss', 'ungram_loss', 'correct'] + dependency_fields
self.model.model._make_test_function()
progbar = Progbar(len(self.deps_test))
for i, dep in enumerate(self.deps_test):
inp = np.zeros((1, self.maxlen))
v = int(dep['verb_index']) - 1
tokens = dep[self.field].split()[:v + 1]
ints = [self.vocab_to_ints[x] for x in tokens]
try:
ungram = self.vocab_to_ints[self.inflect_verb[tokens[v]]]
except KeyError: # reinflected form not in vocabulary: ignore
continue
n = len(ints) - 1
inp[0, -n:] = ints[:-1]
gram_loss = self.model.test_on_batch(inp, np.array([ints[v]]))
ungram_loss = self.model.test_on_batch(inp, np.array([ungram]))
recs.append((gram_loss, ungram_loss, gram_loss < ungram_loss) +
tuple(dep[x] for x in dependency_fields))
if i % 16 == 0:
progbar.update(i)
self.test_results = pd.DataFrame(recs, columns=columns)
def train(self, n_epochs=10):
if not hasattr(self, 'model'):
self.create_model()
self.compile_model()
self.serialize_class_data()
self.serialize_model()
validation_split = 0.1
split_at = int(len(self.X_train) * (1. - validation_split))
x, val_x = self.X_train[:split_at], self.X_train[split_at:]
y, val_y = self.Y_train[:split_at], self.Y_train[split_at:]
training_loss_history = []
validation_loss_history = []
for epoch in range(n_epochs):
print('Epoch', epoch)
training_loss = []
end = int(float(len(x)) / self.batch_size)
progbar = Progbar(end)
for i in range(0, len(x), self.batch_size):
inp = sequence.pad_sequences(x[i:i + self.batch_size],
maxlen=self.maxlen)
out = y[i:i + self.batch_size]
loss = self.model.train_on_batch(inp, out)
training_loss.append(loss)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
# test on validation set
validation_loss = []
print()
print('Evaluating on validation set:')
end = int(float(len(val_x)) / self.batch_size)
progbar = Progbar(end)
for i in range(0, len(val_x), self.batch_size):
inp = sequence.pad_sequences(val_x[i:i + self.batch_size],
maxlen=self.maxlen)
out = val_y[i:i + self.batch_size]
output = self.model.test_on_batch(inp, out)
validation_loss.append(output)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
training_loss_history.append(np.mean(training_loss))
validation_loss_history.append(np.mean(validation_loss))
filename = op.join(self.serialization_dir,
'weights_epoch%d.h5' % epoch)
self.model.save_weights(filename, overwrite=True)
print()
print(('Mean training loss: %5.3f; mean validation loss: %5.3f\n' %
(training_loss_history[-1], validation_loss_history[-1])))
if (len(validation_loss_history) > 1
and validation_loss_history[-1] >=
validation_loss_history[-2]):
break
self.training_history = (list(map(float, training_loss_history)),
list(map(float, validation_loss_history)))
def evaluate(self, howmany=1000):
self.model.model._make_test_function()
random.seed(0)
shuffled = self.deps_test[:]
random.shuffle(shuffled)
shuffled = shuffled[:howmany]
X_test = []
Y_test = []
for dep in shuffled:
tokens = self.process_single_dependency(dep)
ints = []
for token in tokens:
if token not in self.vocab_to_ints:
# zero is for pad
x = self.vocab_to_ints[token] = len(self.vocab_to_ints) + 1
self.ints_to_vocab[x] = token
ints.append(self.vocab_to_ints[token])
first = 1
for i in range(first, len(ints) - 1):
X_test.append(ints[:i])
Y_test.append(ints[i])
test_loss = []
end = int(float(len(X_test) / self.batch_size))
progbar = Progbar(end)
for i in range(0, len(X_test), self.batch_size):
inp = sequence.pad_sequences(X_test[i:i + self.batch_size],
maxlen=self.maxlen)
out = Y_test[i:i + self.batch_size]
output = self.model.test_on_batch(inp, out)
test_loss.append(output)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
return np.mean(test_loss)
X, y = get_data(files[n:n+batch_size], n)
else:
X, y = get_data(files[n:], n)
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0) # , random_state=0
X_train = np.array(X_train)
X_test = np.array(X_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_test = np_utils.to_categorical(y_test, nb_classes)
model.train_on_batch(X_train, Y_train)
l, a = model.test_on_batch(X_test, Y_test)
acc.append(a)
loss.append(l)
print "Epoch: %s <==> Val_loss: %s <> Val_acc: %s "% ( str(e), str(sum(loss) / len(loss)), str(sum(acc) / len(acc)) )
'''
fill between on open and close included:
Epoch: 0 <==> Val_loss: 1.19601917664 <> Val_acc: 0.491666666667
Epoch: 1 <==> Val_loss: 0.895532437166 <> Val_acc: 0.441666666667
Epoch: 2 <==> Val_loss: 0.89461884896 <> Val_acc: 0.408333333333
Epoch: 3 <==> Val_loss: 0.900162645181 <> Val_acc: 0.433333333333
Epoch: 4 <==> Val_loss: 0.908274920781 <> Val_acc: 0.4
Epoch: 5 <==> Val_loss: 0.906020263831 <> Val_acc: 0.45
Epoch: 6 <==> Val_loss: 0.910774775346 <> Val_acc: 0.475
Epoch: 7 <==> Val_loss: 0.922956418991 <> Val_acc: 0.45
b_x = X[b*batch_size:b*batch_size+batch_size] # (b, 50, 69)
b_y = Y[b*batch_size:b*batch_size+batch_size] # (b, 50, 48)
loss, acc = model.train_on_batch(b_x, b_y)
train_loss += loss*b_x.shape[0]
train_acc += acc*b_x.shape[0]
train_loss /= train_len
train_acc /= train_len
valid_loss = 0.0
valid_acc = 0.0
#count = 0
for b in range(valid_batch_num):
b_x = X_valid[b*batch_size:b*batch_size+batch_size] # (b, 50, 69)
b_y = Y_valid[b*batch_size:b*batch_size+batch_size] # (b, 50, 48)
loss, acc = model.test_on_batch(b_x, b_y)
valid_loss += loss*b_x.shape[0]
valid_acc += acc*b_x.shape[0]
valid_loss /= valid_len
valid_acc /= valid_len
#y = model.predict_on_batch(b_x) # (b, 50, 48)
#print(y.shape)
#y = np.reshape(y, (-1, 48))
#b_y = np.reshape(b_y, (-1, 48))
#y = np.argmax(y, axis=1)
#b_y = np.argmax(b_y, axis=1)
#acc += sum(y==b_y)
#count += len(y)
#print("Accuracy:", str(acc/count))
class Model() :
# mode = 'encode' || 'train'
def __init__(self, mode) :
self.io_dim = Codec.n_chars
with open('config/semantic.json') as file :
semantic_config = json.load(file)
self.feature_dim = semantic_config['vectorDim']
self.seq_len = semantic_config['seqLength']
self.mode = mode
self.model = SeqModel()
self.encoder = SeqContainer()
self.encoder.add(TimeDistributedDense(
input_dim = self.io_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid'))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.encoder.add(GRU(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
inner_activation = 'hard_sigmoid',
truncate_gradient = self.seq_len,
return_sequences = False))
self.model.add(self.encoder)
if mode == 'train' :
self.decoder = SeqContainer()
self.decoder.add(SimpleRNN(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.feature_dim,
activation = 'sigmoid',
truncate_gradient = self.seq_len,
return_sequences = True))
self.decoder.add(TimeDistributedDense(
input_dim = self.feature_dim,
input_length = self.seq_len,
output_dim = self.io_dim,
activation = 'sigmoid'))
self.model.add(RepeatVector(self.seq_len, input_shape = (self.feature_dim,)))
self.model.add(self.decoder)
def _load_weights(self, path) :
with h5py.File(path, 'r') as file :
group = file['/weights']
n_layers = group.attrs.get('n_layers')[0]
weights = []
for i in range(n_layers) :
layer_weights = file['/weights/layer_' + str(i)][()]
weights.append(layer_weights)
return weights
def load(self) :
encoder_weights = self._load_weights('data/encoder.hdf5')
self.encoder.set_weights(encoder_weights)
if self.mode == 'train' :
decoder_weights = self._load_weights('data/decoder.hdf5')
self.decoder.set_weights(decoder_weights)
def _save_weights(self, weights, path) :
with h5py.File(path, 'w') as file :
group = file.create_group('weights')
n_layers = len(weights)
group.attrs.create('n_layers', np.array([n_layers]))
for i, layer_weights in enumerate(weights) :
group.create_dataset('layer_' + str(i), data = layer_weights)
def save(self) :
if self.mode != 'train' :
raise Exception('invalid mode')
encoder_weights = self.encoder.get_weights()
decoder_weights = self.decoder.get_weights()
self._save_weights(encoder_weights, 'data/encoder.hdf5')
self._save_weights(decoder_weights, 'data/decoder.hdf5')
def compile(self) :
self.model.compile(loss = 'categorical_crossentropy', optimizer = Adadelta(clipnorm = 1.))
# in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim)
# return train (loss, accuracy)
def train(self, in_data, out_data) :
if self.mode != 'train' :
raise Exception('invalid mode')
return self.model.train_on_batch(in_data, out_data, accuracy = True)
# in_data & out_data numpy bool array of shape (n_sample, seq_len, io_dim)
# return the evaluation (loss, accuracy)
def evaluate(self, in_data, out_data) :
if self.mode != 'train' :
raise Exception('invalid mode')
return self.model.test_on_batch(in_data, out_data, accuracy = True)
# sequence : numpy bool array of shape (seq_len, io_dim)
# return : numpy float32 array of shape (feature_dim)
def encode(self, sequence) :
if self.mode != 'encode' :
raise Exception('invalid mode')
input_sequences = np.ndarray((1, self.seq_len, self.io_dim), dtype = np.bool)
input_sequences[0] = sequence
return self.model.predict(input_sequences)[0]
metrics=['accuracy'])
# In[ ]:
nb_epoch = 1
nb_train_samples = 2048
nb_validation_samples = 832
# In[ ]:
while True:
x_train, y_train = train_generator.next()
print('TRAIN', model.train_on_batch(x_train, y_train))
x_test, y_test = validation_generator.next()
print('TEST', model.test_on_batch(x_test, y_test))
# model.fit_generator(
# train_generator,
# samples_per_epoch=nb_train_samples,
# nb_epoch=nb_epoch,
# validation_data=validation_generator,
# nb_val_samples=nb_validation_samples)
print('hello done.')
# In[ ]:
model.save_weights('models/1000-samples--1-epochs.h5')
# In[ ]:
index += 1
model.save_weights("test_weights.hdf5", overwrite=True)
model.load_weights("test_weights.hdf5")
end = datetime.now()
diff = end - begin
avgSec = diff.total_seconds() / EPOCHS
avgMin = int(avgSec / 60)
avgHour = int(avgMin / 60)
avgDay = int(avgHour / 24)
avgSec -= 60 * avgMin
avgMin -= 60 * avgHour
avgHour -= 24 * avgDay
# loss, acc = model.evaluate(tX, tY, batch_size=BATCH_SIZE, show_accuracy=True)
progbar = generic_utils.Progbar(tX.shape[0])
index = 0
for i in range(len(tX) / BATCH_SIZE):
loss, acc = model.test_on_batch(tX[index : index + BATCH_SIZE], tY[index : index + BATCH_SIZE], accuracy=True)
progbar.add(BATCH_SIZE, values=[("test loss", loss), ("test acc", acc)])
index += BATCH_SIZE
"""
fResult = open('test.txt', 'a+')
fResult.write('Test loss / test accuracy = %.4f / %.4f\n'%(loss, acc))
fResult.write('mode acc / test mode acc = %.4f / %.4f\n'%(accmode, taccmode))
fResult.write('Average learning time = %ddays %d:%d:%d\n\n'%(avgDay, avgHour, avgMin, avgSec))
fResult.close()
"""
class LogRegressionMNIST(Optimizable):
"""
Implementation of a one-layer neural network to perform logistic
regression on the MNIST dataset.
"""
def __init__(self, n_batch: int = 128):
"""
Constructor.
:param n_batch: number of elements in the batch used to compute the
gradients with back-propagation.
"""
self.n_batch = n_batch
self.n_classes = 10
self.img_rows, self.img_cols = 28, 28
self.input_size = self.img_rows * self.img_cols
self.x_train, self.y_train, self.n_train, \
self.x_test, self.y_test, self.n_test =\
None, None, None, None, None, None
self.model = None
self.weights = None
self.weights_shapes = None
self.gradients = None
self.get_keras_gradients = None
self.history = None
self.test_history = None
self.training_history = None
self.dim_observed = 0
self.build_dataset()
return
def build_dataset(self) -> None:
"""
Downloads the MNIST dataset if needed and format it (normalization and
formatting of the y vectors).
"""
# Load the MNIST dataset and reshape it accordingly to the model
(self.x_train, self.y_train), (self.x_test, self.y_test) =\
mnist.load_data()
self.n_train = self.x_train.shape[0]
self.n_test = self.x_test.shape[0]
self.x_train = self.x_train.reshape(self.x_train.shape[0],
self.input_size)
self.x_test = self.x_test.reshape(self.x_test.shape[0],
self.input_size)
# Normalize
self.x_train = self.x_train.astype('float32') / 255.0
self.x_test = self.x_test.astype('float32') / 255.0
# Convert class vectors to binary class matrices
self.y_train = to_categorical(self.y_train, self.n_classes)
self.y_test = to_categorical(self.y_test, self.n_classes)
return
def build_network(self) -> None:
"""
Builds the network architecture using Keras utilities and compiles it.
One hidden layer implementing logistic regression.
"""
self.model = Sequential()
self.model.add(Dense(self.n_classes, activation='sigmoid',
input_dim=self.x_train.shape[1]))
self.model.add(Dropout(0.7))
self.model.compile(loss='categorical_crossentropy', optimizer='sgd',
metrics=['accuracy'])
return
def build_weight_buffers(self) -> None:
"""
Builds some buffers used to access the weights of the network and the
shapes of the layers.
"""
self.weights = self.model.trainable_weights
self.weights_shapes = []
self.dim_observed = 0
for w_vec in self.weights:
self.weights_shapes.append(w_vec.shape)
self.dim_observed += int(np.prod(np.array(w_vec.shape)))
return
def build_gradient_buffers(self) -> None:
"""
Builds some buffers that will be used to compute the gradients of the
loss function with respect to the weights in the network.
"""
self.gradients =\
self.model.optimizer.get_gradients(self.model.total_loss,
self.weights)
input_tensors = [self.model.inputs[0], # input data
self.model.sample_weights[0], # weight for each sample
self.model.targets[0], # labels
k_back.learning_phase()] # train or test mode
self.get_keras_gradients = k_back.function(inputs=input_tensors,
outputs=self.gradients)
return
def build_model(self) -> None:
"""
Builds the whole model by invoking the previous auxiliary functions.
"""
self.build_network()
self.build_weight_buffers()
self.build_gradient_buffers()
self.history = LossHistory()
self.test_history = []
self.training_history = []
return
def next_train_batch(self) -> [np.array, np.array]:
"""
Extracts a new batch from the training set.
:return: list with two elements: the x values of the batch and the
corresponding targets.
"""
index = sample(range(self.n_train), self.n_batch)
x_sample = self.x_train[index, :]
y_sample = self.y_train[index, :]
return [x_sample, y_sample]
def next_test_batch(self) -> [np.array, np.array]:
"""
Extracts a new batch from the test set.
:return: list with two elements: the x values of the batch and the
corresponding targets.
"""
index = sample(range(self.n_test), self.n_batch)
x_sample = self.x_test[index, :]
y_sample = self.y_test[index, :]
return [x_sample, y_sample]
def train(self, n_epochs: int = 5) -> None:
"""
Trains the neural network for a fixed number of epochs using the
internal optimizers of Keras.
:param n_epochs: number of training epochs
"""
self.model.fit(self.x_train, self.y_train, epochs=n_epochs,
batch_size=self.n_batch, callbacks=[self.history])
return
def predict(self, x_new: np.array) -> np.array:
"""
Predicts the label for the input x_new using the current state of the
network.
:param x_new: numpy array with dimensions
[n_items, self.img_rows, self.img_cols] containing the test points.
:return: numpy array containing the predicted labels.
"""
return self.model.predict(x_new, batch_size=self.n_batch)
def compute_test_loss(self) -> None:
"""
Computes the value of the loss function on a randomly chosen test batch.
"""
[x_batch, y_batch] = self.next_test_batch()
loss = self.model.test_on_batch(x_batch, y_batch)
self.history.losses.append(loss)
self.test_history.append(loss)
return
def compute_training_loss(self) -> None:
"""
Computes the value of the loss function on a randomly chosen test batch.
"""
[x_batch, y_batch] = self.next_train_batch()
loss = self.model.test_on_batch(x_batch, y_batch)
self.history.losses.append(loss)
self.training_history.append(loss)
return
def get_weight_vector(self) -> np.array:
"""
Gets the values of the weights of the network, re-formatted as a 1-D
vector.
:return: 1-D numpy array containing the value of the weights.
"""
weights = self.model.get_weights()
n_layers = len(weights)
weights_vector = weights[0].flatten()
for n in range(1, n_layers):
weights_vector = np.append(weights_vector, weights[n].flatten())
return weights_vector
def set_weight_vector(self, weights: np.array) -> None:
"""
Takes the weights vector passed as argument, converts it to a list of
elements with the right shapes and sets the new weights of the network.
:param weights: 1-D vector containing the weight values.
"""
processed_size = 0
new_weights_list = []
for tensor_shape in self.weights_shapes:
present_size = int(np.prod(np.array(tensor_shape)))
vector_end = processed_size + present_size
layer_weights = np.copy(weights[processed_size:vector_end])
new_weights_list.append(np.reshape(layer_weights, tensor_shape))
processed_size = processed_size + present_size
self.model.set_weights(new_weights_list)
return
def get_gradient_vector(self) -> np.array:
"""
Compute the gradient of the loss function of the curent weights and
returns it as a 1D numpy array.
:return: 1D numpy array containing the numerical value of the gradient.
"""
[x_batch, y_batch] = self.next_train_batch()
inputs = [x_batch, np.ones(self.n_batch), y_batch, 0]
grads = self.get_keras_gradients(inputs)
n_layers = len(grads)
grad_vector = grads[0].flatten()
for n in range(1, n_layers):
grad_vector = np.append(grad_vector, grads[n].flatten())
return grad_vector
def f(self, x: np.array) -> float:
"""
Returns the value of the training loss function of the network computed
with the weights in x.
:param x: numpy array containing the weights of the network.
:return: value of the objective function at x.
"""
self.set_weight_vector(x.flatten())
self.compute_training_loss()
return self.training_history[-1]
def get_gradient(self, x: np.array) -> np.array:
"""
Returns the gradients of the objective function we want to optimize
computed at x.
:param x: numpy array containing the desired coordinates.
:return: function gradients at x.
"""
self.set_weight_vector(x.flatten())
gradient = self.get_gradient_vector()
return gradient
hist = model.fit(origin,
target,
epochs=numepoch,
batch_size=batchsize,
validation_split=0.01,
sample_weight=data_weights,
shuffle=True,
callbacks=[history]) # starts training
stop = timeit.default_timer()
print('time elapsed => ', stop - start)
model.save_weights('myweights.h5')
#print(history.losses);
np.savetxt('loss.txt', history.losses, delimiter=' ')
print("test loss ", model.test_on_batch(origintest, targettest))
# test error
#testLoss = model.test_on_batch(origintest,targettest);
#print("test loss ",model.evaluate(origintest,targettest,batch_size = None,verbose = 1));
filepath = 'type' + str(nodetypt) + '.txt'
with open(filepath, 'wb') as f:
lent = len(model.layers)
f.write((str(lent) + "\n").encode())
for layer in model.layers:
weights = layer.get_weights()
parameter_len = len(weights)
f.write((str(parameter_len) + "\n").encode())
for weight in weights:
header_ = ""
for s in weight.shape:
print('Test score:', score)
else:
print('Using real time data augmentation')
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print('Training...')
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(num_images-test_size)
for X_batch, Y_batch in flow(image_list[0:-test_size]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
loss = model.train_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('train loss', loss)])
print('Testing...')
# test time!
progbar = generic_utils.Progbar(test_size)
for X_batch, Y_batch in flow(image_list[-test_size:]):
X_batch = X_batch.reshape(X_batch.shape[0], 3, img_rows, img_cols)
Y_batch = np_utils.to_categorical(Y_batch, nb_classes)
score = model.test_on_batch(X_batch, Y_batch)
progbar.add(X_batch.shape[0], values=[('test loss', score)])
json_string = model.to_json()
open('cnn1_model_architecture.json', 'w').write(json_string)
model.save_weights('cnn1_model_weights.h5')
def run():
datadir = "/reg/d/ana01/temp/davidsch/ImgMLearnFull"
h5files = glob(os.path.join(datadir, "amo86815_mlearn-r070*.h5"))
h5files.extend(glob(os.path.join(datadir, "amo86815_mlearn-r071*.h5")))
# h5files = ["/reg/d/ana01/temp/davidsch/ImgMLearnFull/amo86815_mlearn-r071-c0000.h5",
# "/reg/d/ana01/temp/davidsch/ImgMLearnFull/amo86815_mlearn-r071-c0001.h5",
# "/reg/d/ana01/temp/davidsch/ImgMLearnFull/amo86815_mlearn-r071-c0002.h5"]
assert len(h5files)>0
datareader = H5MiniBatchReader(h5files=h5files,
minibatch_size=32,
validation_size=64,
feature_dataset='xtcavimg',
label_dataset='acq.peaksLabel',
return_as_one_hot=True,
feature_preprocess=['log','mean'],
number_of_batches=None,
class_labels_max_imbalance_ratio=1.0,
add_channel_to_2D='channel_row_column',
max_mb_to_preload_all=None,
random_seed=None,
verbose=True)
validation_features, validation_labels = datareader.get_validation_set()
print("starting to build and compile keras/theano model...")
sys.stdout.flush()
t0 = time.time()
model = Sequential()
## layer 1
kern01_W_init = (0.06/2.0)*scipy.stats.truncnorm.rvs(-2.0, 2.0, size=(8,1,8,8)).astype(np.float32)
kern01_B_init = np.zeros(8,dtype=np.float32)
model.add(Convolution2D(8,8,8, border_mode='same', weights=[kern01_W_init, kern01_B_init],
input_shape=datareader.features_placeholder_shape()[1:]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(10,10), strides=(13,13)))
## layer 2
kern02_W_init = (0.06/2.0)*scipy.stats.truncnorm.rvs(-2.0, 2.0, size=(8,8,6,6)).astype(np.float32)
kern02_B_init = np.zeros(8,dtype=np.float32)
model.add(Convolution2D(8,6,6, border_mode='same', weights=[kern02_W_init, kern02_B_init]))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(10,10), strides=(13,13)))
model.add(Flatten())
## layer 3
H03_W_init = (0.06/2.0)*scipy.stats.truncnorm.rvs(-2.0, 2.0, size=(96,16)).astype(np.float32)
H03_B_init = np.zeros(16,dtype=np.float32)
model.add(Dense(16, weights=[H03_W_init, H03_B_init]))
model.add(Activation('relu'))
## layer 4
H04_W_init = (0.06/2.0)*scipy.stats.truncnorm.rvs(-2.0, 2.0, size=(16, datareader.num_outputs())).astype(np.float32)
H04_B_init = np.zeros(datareader.num_outputs(),dtype=np.float32)
model.add(Dense(datareader.num_outputs(), weights=[H04_W_init, H04_B_init]))
model.add(Activation('softmax'))
sgd = SGD(lr=0.01, decay=0.0004, momentum=0.96)
model.compile(loss='categorical_crossentropy', optimizer=sgd)
print("building/compiling theano model took %.2f sec" % (time.time()-t0),)
sys.stdout.flush()
for step_number in range(3000):
t0 = time.time()
train_features, train_labels = datareader.get_next_minibatch()
model.train_on_batch(train_features, train_labels)
print("step %3d took %.2f sec." % (step_number, time.time()-t0))
sys.stdout.flush()
print("Starting evaluation.")
t0 = time.time()
loss, validation_accuracy = model.test_on_batch(validation_features, validation_labels, accuracy=True, sample_weight=None)
print("validation accuracy: %.2f%%" % (100.0*validation_accuracy,))
print("evaluation took %.2f sec" % (time.time()-t0,))
model.compile(loss=qri.mae_clip, optimizer=sgd)
# Use early stopping and saving as callbacks
early_stop = EarlyStopping(monitor='val_loss', patience=10)
save_best = ModelCheckpoint("models/%s.mdl" % MDL_NAME, save_best_only=True)
callbacks = [early_stop, save_best]
# Train model
t0 = time.time()
hist = model.fit(train_set[0], train_set[1], validation_data=valid_set,
verbose=2, callbacks=callbacks, nb_epoch=1000, batch_size=20)
time_elapsed = time.time() - t0
# Load best model
model.load_weights("models/%s.mdl" % MDL_NAME)
# Print time elapsed and loss on testing dataset
test_set_loss = model.test_on_batch(test_set[0], test_set[1])
print "\nTime elapsed: %f s" % time_elapsed
print "Testing set loss: %f" % test_set_loss
# Save results
qri.save_results("results/%s.out" % MDL_NAME, time_elapsed, test_set_loss)
qri.save_history("models/%s.hist" % MDL_NAME, hist.history)
# Plot training and validation loss
qri.plot_train_valid_loss(hist.history)
# Make predictions
qri.plot_test_predictions(model, train_set)
def runCNN_Train(feature_path,
fold='config2_fold_1',
useType=['Spec', 'Ceps', 'GCoS'],
use_time_range=None):
############### Load and Preprocess Data ###############
numVal = 30 #number of validation songs
numTrain = 180 #number of training songs
use_same_val = True #weither to use the same validation set as last run
use_same_train = True #wether to use the same training set as last run
numFrame = 5 #augment the time dimension of 5 (0.05s)
val_fold = 'val_' + fold + '.txt'
train_fold = 'train_' + fold + '.txt'
#val_fold = Hack4LoadbyFold(numVal, 'test_%s.txt' % fold, 'val.txt', use_same=use_same_val) #you can specify your own fold
#train_fold = Hack4LoadbyFold(numTrain, 'train_%s.txt' % fold, 'train.txt', use_same=use_same_train)
print('Loading training data')
train_data, train_label, train_index, tData_shape = LoadbyFold(
train_fold, feature_path, use_type=useType, t_range=use_time_range)
t_samples = tData_shape[0]
print('\nLoading validation data')
val_data, val_label, val_index, vData_shape = LoadbyFold(val_fold,
feature_path,
use_type=useType)
v_samples = vData_shape[0]
data_shape = (numFrame, tData_shape[2], tData_shape[3])
print('\nInput shape: %s' % str(data_shape))
############### Define Model ###############
initLR = 0.001
num_middle_node = 512
k_init = keras.initializers.RandomNormal(mean=0.0, stddev=0.05)
model_name = 'CNN_model_(' + fold + ')_' + str(useType) + '_' + str(
use_time_range)
model = Sequential()
model.add(
Conv2D(32, (5, 3),
activation='selu',
input_shape=data_shape,
kernel_initializer=k_init))
model.add(Conv2D(32, (1, 3), activation='selu', kernel_initializer=k_init))
model.add(MaxPooling2D(pool_size=(1, 2)))
model.add(Flatten())
model.add(Dropout(0.5))
model.add(
Dense(num_middle_node, activation='selu', kernel_initializer=k_init))
model.add(Dropout(0.5))
model.add(
Dense(num_middle_node, activation='selu', kernel_initializer=k_init))
model.add(Dropout(0.5))
model.add(Dense(88, activation='sigmoid'))
optim = keras.optimizers.Adam(lr=initLR)
model.compile(loss=keras.losses.binary_crossentropy,
optimizer=optim,
metrics=[st.Precision, st.Recall, st.Fscore])
############### Training Field ###############
epoch = 20 #max epochs for training
batchSize = 100
v_batchSize = 500
break_patience = 4 #early stop epochs. If val_loss doesn't decrease for given epochs, the training will stop.
t_idx = np.random.permutation(t_samples)
v_idx = np.random.permutation(v_samples)
train_batches = int(np.ceil(t_samples / batchSize))
val_batches = int(np.ceil(v_samples / v_batchSize))
history = {}
history['loss'] = []
history['Fscore'] = []
history['val_loss'] = []
history['val_Fscore'] = []
patience = 0
best_f = 0
best_v = 100000
best_epoch = 0
print('Using ' + str(useType) + ' for training')
print('\nTrain on %d samples - Validate on %d samples' %
(t_samples, v_samples))
for e in range(epoch):
cur_ep = 'Epoch %d/%d - ' % (e + 1, epoch)
loss = 0
fscore = 0
recall = 0
precision = 0
for i in range(train_batches):
sel_t = t_idx[i * batchSize:(i + 1) * batchSize]
if i == train_batches - 1:
sel_t = t_idx[i * batchSize:]
data, label = preCNN_processBatch(train_data, train_label,
train_index, sel_t, numFrame)
scalar = model.train_on_batch(data, label)
loss += scalar[0]
fscore += scalar[3]
recall += scalar[2]
precision += scalar[1]
batch_info = cur_ep + '%d/%d - ' % (i + 1, train_batches)
batch_info += 'loss: %.4f - Precision: %.4f - Recall: %.4f - Fscore: %.4f' % (
loss / (i + 1), precision / (i + 1), recall / (i + 1), fscore /
(i + 1))
if i == (train_batches - 1):
print(batch_info)
history['loss'].append(loss / train_batches)
history['Fscore'].append(fscore / train_batches)
else:
print(batch_info, end='\r')
### Validation ###
v_loss = 0
v_fscore = 0
for i in range(val_batches):
print('Validation progress: %d/%d' % (i + 1, val_batches),
end='\r')
sel_v = v_idx[i * v_batchSize:(i + 1) * v_batchSize]
if i == val_batches - 1:
sel_v = v_idx[i * v_batchSize:]
data, label = preCNN_processBatch(val_data, val_label, val_index,
sel_v, numFrame)
#data = preCNN(combineFrame_2(val_data, numFrame, sample_per_song=3000, sel_t), channel=3)[:,:,:,use_specific_channel]
v_scalar = model.test_on_batch(data, label)
v_loss += v_scalar[0]
v_fscore += v_scalar[3]
print('Validation - loss: %.4f - F-score: %.4f' %
(v_loss / val_batches, v_fscore / val_batches))
print(
'-------------------------------------------------------------------------------------------'
)
history['val_loss'].append(v_loss / val_batches)
history['val_Fscore'].append(v_fscore / val_batches)
### Early Stop Check (according to val_loss) ###
if v_loss >= best_v:
patience += 1
else:
patience = 0
best_epoch = e
best_v = v_loss
best_f = v_fscore
model.save('./Result/%s.hdf5' % model_name)
if patience == break_patience:
print('Early Stopped')
break
info = 'Best epoch: %d - Loss: %.4f - F-score: %.4f\n' % (best_epoch + 1, (
best_v / val_batches), (best_f / val_batches))
print(info)
log = open('log.txt', 'a')
log.write(fold + str(useType) + '\n')
log.write(info + '\n\n')
log.close()
#Save training and validation history
out_history = h5py.File(
"History/CNN_history(" + fold + str(useType) + ").hd5", 'w')
out_history.create_dataset('loss', data=history['loss'])
out_history.create_dataset('Fscore', data=history['Fscore'])
out_history.create_dataset('val_loss', data=history['val_loss'])
out_history.create_dataset('val_Fscore', data=history['val_Fscore'])
out_history.close()
return info, model
def CNN():
input_frames=10
batch_size = 32
nb_classes = 20
nb_epoch = 200
img_rows, img_cols = 224,224
img_channels = 2*input_frames
print 'X_sample: '+str(X_sample.shape)
print 'X_test: '+str(X_test.shape)
print 'Y_test: '+str(Y_test.shape)
print 'Preparing architecture...'
model = Sequential()
model.add(Convolution2D(96, 7, 7, border_mode='same',input_shape=(img_channels, img_rows, img_cols)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(256, 5, 5, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(512, 3, 3, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(512, 3, 3, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Convolution2D(512, 3, 3, border_mode='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(2048))
fc_output=Activation('relu')
model.add(fc_output)
model.add(Dropout(0.5))
model.add(Dense(2048))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
softmax_output=Activation('softmax')
model.add(softmax_output)
print 'Starting with training...'
gc.collect()
sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss={'output1':'categorical_crossentropy'}, optimizer=sgd)
print("Using real time data augmentation")
datagen = ImageDataGenerator(
featurewise_center=True, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=True, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # apply ZCA whitening
rotation_range=20, # randomly rotate images in the range (degrees, 0 to 180)
width_shift_range=0.2, # randomly shift images horizontally (fraction of total width)
height_shift_range=0.2, # randomly shift images vertically (fraction of total height)
horizontal_flip=True, # randomly flip images
vertical_flip=True) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_sample)
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_train, Y_train in getTrainData():
for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size=batch_size):
loss = model.train_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("train loss", loss[0]),("train accuracy", loss[1])])
fc_output
softmax_output
print('Saving layer representation and saving weights...')
with h5py.File('fc_output.h5', 'w') as hf:
hf.create_dataset('fc_output', data=fc_output)
with h5py.File('softmax_output.h5', 'w') as hf:
hf.create_dataset('softmax_output', data=softmax_output)
model.save_weights('temporal_stream_model.h5')
print("Testing...")
# test time!
progbar = generic_utils.Progbar(X_test.shape[0])
for X_test, Y_test in getTestData():
for X_batch, Y_batch in datagen.flow(X_test, Y_test, batch_size=batch_size):
score = model.test_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("test loss", score[0]),("test accuracy", score[1])])
height_shift_range=0, # randomly shift images vertically (fraction of total height)
horizontal_flip=False, # randomly flip images
vertical_flip=False) # randomly flip images
# compute quantities required for featurewise normalization
# (std, mean, and principal components if ZCA whitening is applied)
datagen.fit(X_train)
for e in range(nb_epoch):
print('-'*40)
print('Epoch', e)
print('-'*40)
print("Training...")
# batch train with realtime data augmentation
progbar = generic_utils.Progbar(X_train.shape[0])
for X_batch, Y_batch in datagen.flow(X_train, Y_train, batch_size=batch_size):
score, trainAccu = model.train_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("train accuracy", trainAccu)])
print("Testing...")
# test time!
progbar = generic_utils.Progbar(X_test.shape[0])
for X_batch, Y_batch in datagen.flow(X_test, Y_test, batch_size=batch_size):
score, testAccu = model.test_on_batch(X_batch, Y_batch, accuracy=True)
progbar.add(X_batch.shape[0], values=[("test accuracy", testAccu)])
trainScores.append(trainAccu)
testScores.append(testAccu)
scipy.io.savemat('cnn_results', {'trainAccu': trainScores, 'testAccu': testScores})
print ('Average train accuracies: {0}'.format(np.mean(trainScores)))
print ('Average test accuracies: {0}'.format(np.mean(testScores)))
