def test_nested_sequential(in_tmpdir):
(x_train, y_train), (x_test, y_test) = _get_test_data()
inner = Sequential()
inner.add(Dense(num_hidden, input_shape=(input_dim,)))
inner.add(Activation('relu'))
inner.add(Dense(num_class))
middle = Sequential()
middle.add(inner)
model = Sequential()
model.add(middle)
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test))
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=2, validation_split=0.1)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=0)
model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, shuffle=False)
model.train_on_batch(x_train[:32], y_train[:32])
loss = model.evaluate(x_test, y_test, verbose=0)
model.predict(x_test, verbose=0)
model.predict_classes(x_test, verbose=0)
model.predict_proba(x_test, verbose=0)
fname = 'test_nested_sequential_temp.h5'
model.save_weights(fname, overwrite=True)
inner = Sequential()
inner.add(Dense(num_hidden, input_shape=(input_dim,)))
inner.add(Activation('relu'))
inner.add(Dense(num_class))
middle = Sequential()
middle.add(inner)
model = Sequential()
model.add(middle)
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate(x_test, y_test, verbose=0)
assert(loss == nloss)
# test serialization
config = model.get_config()
Sequential.from_config(config)
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
def move_grade_training():
"""Train a model to predict if a given move is 'good' or not"""
WHOSE_TURN_DIM = 1 #i.e. X or O
BOARD_DIM = 81 #i.e. 9x9
POSS_MOVE_DIM = 81 #ie. same as board size
INPUT_DIM = WHOSE_TURN_DIM + BOARD_DIM + POSS_MOVE_DIM + POSS_MOVE_DIM #turn, board, last_move, new_move
NB_EPOCH = 5
#NOTE: X_PIECE always went first in the training data
model = Sequential()
model.add(Dense(2 * INPUT_DIM, input_dim=INPUT_DIM))
model.add(Dense(INPUT_DIM))
model.add(Dense(BOARD_DIM))
model.add(Dense(1)) #predicting if the move was a 'winning' move or not
model.add(Activation('softmax'))
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])
for batch in training_batch_generator():
X = []
y = []
for game, winner in batch:
#skipping ties for now, as we only want to track winning and losing moves
if winner == constants.NO_PIECE:
continue
#all even values of i represent moves by X_PIECE
#we are starting with an empty board, and hence, all zeros for previous move (i.e. there was no previous move)
prev_move = np.zeros(81, dtype='int32')
for i, board in enumerate(game[:-1]):
#case 1, X won and this is an x move we are scoring
if i % 2 == 0 and winner == constants.X_PIECE:
y.append(1)
#case 2, O won and this is an O move we are scoring
elif (i % 2 == 1) and winner == constants.O_PIECE:
y.append(1)
else:
#this is a loser's move
y.append(0)
turn = i % 2
x1 = np.asarray(game[i].flatten()) #board
x2 = prev_move
row, col = get_move_for_boards(game[i], game[i + 1])
move_idx = np.ravel_multi_index((row, col), (9, 9))
x3 = to_categorical([move_idx], BOARD_DIM)[0]
X.append(np.hstack([[turn], x1, x2, x3]))
#we need to update the previous move for next iteration
prev_move = x3
X = np.asarray(X)
y = np.asarray(y)
model.fit(X, y, batch_size=32, nb_epoch=NB_EPOCH, validation_split=0.05)
with open('keras_model.json', 'w') as f:
f.write(model.to_json())
def test_nested_sequential():
(X_train, y_train), (X_test, y_test) = _get_test_data()
inner = Sequential()
inner.add(Dense(nb_hidden, input_shape=(input_dim,)))
inner.add(Activation("relu"))
inner.add(Dense(nb_class))
middle = Sequential()
middle.add(inner)
model = Sequential()
model.add(middle)
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(X_test, y_test))
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=2, validation_split=0.1)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)
model.train_on_batch(X_train[:32], y_train[:32])
loss = model.evaluate(X_test, y_test, verbose=0)
model.predict(X_test, verbose=0)
model.predict_classes(X_test, verbose=0)
model.predict_proba(X_test, verbose=0)
fname = "test_nested_sequential_temp.h5"
model.save_weights(fname, overwrite=True)
inner = Sequential()
inner.add(Dense(nb_hidden, input_shape=(input_dim,)))
inner.add(Activation("relu"))
inner.add(Dense(nb_class))
middle = Sequential()
middle.add(inner)
model = Sequential()
model.add(middle)
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy", optimizer="rmsprop")
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate(X_test, y_test, verbose=0)
assert loss == nloss
# test serialization
config = model.get_config()
new_model = Sequential.from_config(config)
model.summary()
json_str = model.to_json()
new_model = model_from_json(json_str)
yaml_str = model.to_yaml()
new_model = model_from_yaml(yaml_str)
def test_recursive():
# test layer-like API
graph = Graph()
graph.add_input(name='input1', input_shape=(32,))
graph.add_node(Dense(16), name='dense1', input='input1')
graph.add_node(Dense(4), name='dense2', input='input1')
graph.add_node(Dense(4), name='dense3', input='dense1')
graph.add_output(name='output1', inputs=['dense2', 'dense3'],
merge_mode='sum')
seq = Sequential()
seq.add(Dense(32, input_shape=(32,)))
seq.add(graph)
seq.add(Dense(4))
seq.compile('rmsprop', 'mse')
seq.fit(X_train_graph, y_train_graph, batch_size=10, nb_epoch=10)
loss = seq.evaluate(X_test_graph, y_test_graph)
# test serialization
config = seq.get_config()
new_graph = Sequential.from_config(config)
seq.summary()
json_str = seq.to_json()
new_graph = model_from_json(json_str)
yaml_str = seq.to_yaml()
new_graph = model_from_yaml(yaml_str)
def train_48calibration_net(X_train, y_train):
print (X_train.shape,y_train.shape)
Y_train = np_utils.to_categorical(y_train, nb_classes)
X_train = X_train.astype('float32')
X_train /= 255
model = Sequential()
model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
border_mode='valid',
input_shape=(3, img_rows, img_cols)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool),strides=(2,2)))
#model.add(BatchNormalization(mode=2))
model.add(Convolution2D(nb_filters, nb_conv, nb_conv))
#model.add(BatchNormalization(mode=2))
model.add(Flatten())
model.add(Dense(256))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch,
verbose=1, validation_split=0.2)
json_string = model.to_json()
open('../model/48calibration_architecture.json', 'w').write(json_string)
model.save_weights('../model/48calibration_weights.h5')
def run_mlp(**args):
print("building mlp model:")
print(args["training_data"].shape[0])
print(args["training_data"].shape[1])
model = Sequential()
model.add(Dense(output_dim=512, input_dim=args["training_data"].shape[1], activation="relu"))
# model.add(Dense(output_dim=64, input_dim=128, activation='relu'))
# model.add(Dense(output_dim=32, input_dim=64, activation='relu'))
model.add(Dense(1))
model.add(Activation("linear"))
model.compile(loss="mse", optimizer="rmsprop")
model.fit(args["training_data"], args["training_label"], nb_epoch=50, batch_size=512)
# pickle.dump(model, open('mlp_testmodel.p', 'w'), protocol=4)
json_string = model.to_json()
open("mlp_model_architecture.json", "w").write(json_string)
model.save_weights("mlp_model_weights.h5", overwrite=True)
# output = model.evaluate(args['test_data'], args['test_label'], batch_size=512)
output = model.predict(args["test_data"], verbose=1, batch_size=512)
output_int = list(map(lambda e: int(e), np.round(output)))
pickle.dump(output_int, open("mlp_output.p", "wb"), protocol=4)
return output_int
def train():
X, Y = load_data()
# create model
model = Sequential()
# input_dim is the feature number 8
#model.add(Dense(12, input_dim=8, init='uniform', activation='relu'))
model.add(Dense(256, input_dim=8, init='uniform', activation='relu'))
model.add(Dense(16, init='uniform', activation='relu'))
model.add(Dense(1, init='uniform', activation='sigmoid'))
# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
# Fit the model
#model.fit(X, Y, nb_epoch=150, batch_size=10)
model.fit(X, Y, nb_epoch=1000, batch_size=32, shuffle=True)
# serialize model to JSON
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("model.h5")
print("Saved model to disk")
def create_model(X_train, Y_train):
"""create_model will create a very simple neural net model and save the weights in a predefined directory.
Args:
X_train: Input X_train
Y_train: Lables Y_train
"""
xin = X_train.shape[1]
model = Sequential()
model.add(Dense(units=4, input_shape=(xin, )))
model.add(Activation('tanh'))
model.add(Dense(4))
model.add(Activation('linear'))
model.add(Dense(1))
rms = kop.RMSprop()
print('compiling now..')
model.compile(loss='mse', optimizer=rms)
model.fit(X_train, Y_train, epochs=1000, batch_size=1, verbose=2)
score = model.evaluate(X_train, Y_train, batch_size=1)
print("Evaluation results:", score)
open('pickles/my_model_architecture.json', 'w').write(model.to_json())
print("Saving weights in: ./pickles/my_model_weights.h5")
model.save_weights('pickles/my_model_weights.h5')
def train_model(self):
print '=======begin to prepare data at ' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '========='
list_sorted = self.word2vec()
self.y = np.array(list(self.y))
self.x = list(sequence.pad_sequences(list(self.x), maxlen=max_len))
self.x = np.array(list(self.x))
print '=======end to prepare data at ' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '========='
print '=======begin to train model at ' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '========='
model = Sequential()
model.add(Embedding(input_dim=len(list_sorted) + 1, output_dim=256, input_length=max_len))
model.add(LSTM(128))
model.add(Dropout(0.5))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
model.fit(self.x, self.y, batch_size=16, nb_epoch=10)
json_string = model.to_json()
open('sa_model_architecture.json', 'w').write(json_string)
model.save_weights('sa_model_weights.h5', overwrite=True)
print '=======end to train model at ' + datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + '========='
return model
def main(args=sys.argv[1:]):
model = Sequential()
model.add(Embedding(vocsize + 1, sizes[0], mask_zero=True,
input_length=seq_length))
for size in sizes[:-1]:
model.add(LSTM(128, return_sequences=True))
model.add(LSTM(sizes[-1]))
model.add(Dense(vocsize))
model.add(Activation('softmax'))
print('x.shape:', x.shape)
print('y.shape:', y.shape)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop')
with open('topology.json', 'w') as f:
f.write(model.to_json())
for iteration in range(1, num_batches//batches_per_it):
print()
print('-' * 50)
print('Iteration', iteration)
model.fit(x, y, batch_size=batch_size,
nb_epoch=batches_per_it, verbose=True)
model.save_weights('brain-{}.h5'.format(iteration))
def run_mlp(**args):
print("building mlp model:")
print(args['training_data'].shape[0])
print(args['training_data'].shape[1])
model = Sequential()
model.add(Dense(output_dim=512, input_dim=args['training_data'].shape[1], activation='relu'))
model.add(Dense(1))
model.add(Activation('linear'))
model.compile(loss='mse', optimizer='rmsprop')
model.fit(args['training_data'], args['training_label'], nb_epoch=20, batch_size=512)
json_string = model.to_json()
open('mlp_model_architecture.json', 'w').write(json_string)
model.save_weights(args['output_weight_filename'], overwrite=True)
output = model.predict(args['test_data'], verbose=1, batch_size=512)
if (args['output_type']=='int'):
output_int = list(map(lambda e:int(e), np.round(output)))
pickle.dump(output_int, open(args['output_feat_filename'], 'wb'), protocol=4)
return output_int
else:
pickle.dump(output, open(args['output_feat_filename'], 'wb'), protocol=4)
return output
def trainModel():
inputs, correctOutputs = getNNData()
print("Collected data")
trainingInputs = inputs[:len(inputs)//2]
trainingOutputs = correctOutputs[:len(correctOutputs)//2]
testInputs = inputs[len(inputs)//2:]
testOutputs = correctOutputs[len(correctOutputs)//2:]
model = Sequential()
model.add(Dense(24, input_shape=(24, )))
model.add(Activation('tanh'))
model.add(Dense(24))
model.add(Activation('tanh'))
model.add(Dense(5))
model.add(Activation('softmax'))
model.summary()
model.compile(loss='mean_squared_error', optimizer=SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True))
model.fit(trainingInputs, trainingOutputs, validation_data=(testInputs, testOutputs))
score = model.evaluate(testInputs, testOutputs, verbose=0)
print(score)
json_string = model.to_json()
open('my_model_architecture.json', 'w').write(json_string)
model.save_weights('my_model_weights.h5', overwrite=True)
def make_model(model_dir, input_dim):
print input_dim
model = Sequential()
model.add(Dense(2048, input_dim=input_dim, activation="relu"))
model.add(Dropout(0.3))
model.add(Dense(2048, activation="relu"))
model.add(Dropout(0.3))
model.add(Dense(2048, activation="relu"))
model.add(Dropout(0.3))
model.add(Dense(1))
# model.add(LSTM(512, return_sequences=True,
# input_shape=(None, input_dim)))
# model.add(LSTM(1))
# model.add(Dense(1))
# model.add(TimeDistributedDense(1024, input_dim=1530))
# model.add(LSTM(1024, activation='relu', return_sequences=False))
# model.add(Dropout(0.3))
model.summary()
model_path = model_dir + "/model.json"
with open(model_path, 'w') as f:
f.write(model.to_json())
return model
def create_neural_network(self):
model = Sequential()
model.add(Dense(100, input_dim=self.nstates, activation='relu'))
model.add(Dense(100, activation='relu'))
model.add(Dense(self.nactions,activation='linear'))
#get second last layer of the model, abondon the last layer
layer = model.layers[-2]
nb_action = model.output._keras_shape[-1]
#layer y has a shape(nb_action+1)
#y[:,0] represents V(s;theta)
#y[:,1] represents A(a;theta)
y = Dense(nb_action+1, activation='linear')(layer.output)
#calculate the Q(s,a,;theta)
#dueling type average -> Q(s,a;theta) = V(s;theta) + (A(s,a;theta)-Average_a(A(s,a;theta)))
#outputlayer = Lambda(lambda a:K.expand_dims(a[:,0], -1) + a[:,1:] - K.mean(a[:,1:], keepdims=True), output_shape=(nb_action,))(y)
#dueling type max -> Q(s,a;theta) = V(s;theta) + (A(s,a;theta)-Max_a(A(s,a;theta)))
outputlayer = Lambda(lambda a:K.expand_dims(a[:,0], -1) + a[:,1:] - K.max(a[:,1:,], keepdims=True), output_shape=(nb_action,))(y)
#dueling type naive -> Q(s,a;theta) = V(s;theta) + A(s,a;theta)
#outputlayer = Lambda(lambda a: K.expand_dims(a[:,0], -1) + a[:,1:], output_shape=(nb_action,))(y)
#connect
model = Model(input=model.input, output=outputlayer)
model.compile(loss='mse', optimizer=Adam(lr=self.alpha))
model_json = model.to_json()
with open('cartpole.json','w') as json_file:
json_file.write(model_json)
return model
def load_model(model_name, weights_path=None, model_caption=None):
print("Loading model")
if not model_caption:
model_caption = image_caption_model
image_caption_json = 'models/'+dataname+'/image_captioning_' + model_name + 'model_'+str(model_caption)+\
'_output_rnn_'+str(output_rnn_dim)+'.json'
if weights_path: # Second tim run or predict mode
model = model_from_json(open(image_caption_json).read())
model.load_weights(weights_path)
else:
if image_caption_model == 1:
image_model = Sequential()
image_model.add(Dense(word_vec_dim, input_dim=img_dim))
image_model.add(Reshape((1,word_vec_dim)))
language_model = Sequential()
language_model.add(Embedding(vocab_size, word_vec_dim, input_length=max_caption_len))
language_model.add(GRU(output_dim=word_vec_dim, return_sequences=True))
language_model.add(TimeDistributedDense(word_vec_dim))
model = Sequential()
model.add(Merge([image_model, language_model], mode='concat', concat_axis=1))
model.add(getattr(layers, model_name)(output_rnn_dim, return_sequences=False, input_shape=(max_caption_len+1, word_vec_dim),stateful=False))
model.add(Dense(vocab_size, activation='softmax'))
elif image_caption_model == 2:
image_model1 = Sequential()
image_model1.add(Dense(150, input_dim=img_dim))
image_model1.add(RepeatVector(max_caption_len))
image_model2 = Sequential()
image_model2.add(Dense(450, input_dim=img_dim))
image_model2.add(Reshape((1,450)))
language_model = Sequential()
language_model.add(Embedding(vocab_size, 300, input_length=max_caption_len))
language_model.add(GRU(output_dim=300, return_sequences=True))
language_model.add(TimeDistributedDense(300))
model1 = Sequential()
model1.add(Merge([image_model1, language_model],mode='concat', concat_axis=-1))
model = Sequential()
model.add(Merge([image_model2, model1], mode='concat', concat_axis=1))
model.add(getattr(layers, model_name)(output_rnn_dim, input_shape=(max_caption_len+1, 450),
return_sequences=False, stateful=False))
model.add(Dense(vocab_size, activation='softmax'))
json_string = model.to_json()
open(image_caption_json, 'w').write(json_string)
opt = Adagrad()
model.compile(loss='categorical_crossentropy', optimizer=opt)
return model
def test_batch_input_shape_serialization():
model = Sequential()
model.add(embeddings.Embedding(2, 2,
mask_zero=True,
input_length=2,
batch_input_shape=(2, 2)))
json_data = model.to_json()
reconstructed_model = model_from_json(json_data)
assert(reconstructed_model.input_shape == (2, 2))
def test_merge_sum():
(X_train, y_train), (X_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)
loss = model.evaluate([X_test, X_test], y_test, verbose=0)
model.predict([X_test, X_test], verbose=0)
model.predict_classes([X_test, X_test], verbose=0)
model.predict_proba([X_test, X_test], verbose=0)
# test weight saving
fname = 'test_merge_sum_temp.h5'
model.save_weights(fname, overwrite=True)
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.load_weights(fname)
os.remove(fname)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
nloss = model.evaluate([X_test, X_test], y_test, verbose=0)
assert(loss == nloss)
# test serialization
config = model.get_config()
Sequential.from_config(config)
model.summary()
json_str = model.to_json()
model_from_json(json_str)
yaml_str = model.to_yaml()
model_from_yaml(yaml_str)
class MNISTClassifier(object):
def __init__(self, batch_size=128, nb_classes=10):
self.img_rows, self.img_cols = 28, 28
self.batch_size = batch_size
self.nb_classes = nb_classes
def _loadAndPreprocessTraining(self):
(self.X_train, self.y_train), (self.X_test, self.y_test) = mnist.load_data()
self.X_train = self.X_train.reshape(self.X_train.shape[0], 1, self.img_rows, self.img_cols).astype('float32') / 255
self.X_test = self.X_test.reshape(self.X_test.shape[0], 1, self.img_rows, self.img_cols).astype('float32') / 255
self.Y_train = np_utils.to_categorical(self.y_train, self.nb_classes)
self.Y_test = np_utils.to_categorical(self.y_test, self.nb_classes)
def buildNN(self):
if os.path.isfile("MNISTArchitecture.json"):
print(" loading architecture")
self.model = model_from_json(open('MNISTArchitecture.json').read())
else:
self.model = Sequential()
self.model.add(Convolution2D(64, 5, 5, input_shape=(1, self.img_rows, self.img_cols)))
self.model.add(Activation('relu'))
self.model.add(MaxPooling2D(pool_size=(2, 2)))
self.model.add(Flatten())
self.model.add(Dense(256))
self.model.add(Activation('relu'))
self.model.add(Dropout(0.5))
self.model.add(Dense(self.nb_classes))
self.model.add(Activation('softmax'))
mnistArchitecture = self.model.to_json()
open('MNISTArchitecture.json', 'w').write(mnistArchitecture)
pass
self.model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=0.0000001, momentum=0.8, nesterov=True), metrics=['accuracy'])
if os.path.isfile("MNISTWeights.h5"):
print(" loading weights")
self.model.load_weights('MNISTWeights.h5')
def train(self, nb_epoch=24):
self._loadAndPreprocessTraining()
self.model.fit(self.X_train, self.Y_train, batch_size=self.batch_size, nb_epoch=nb_epoch, verbose=1, validation_data=(self.X_test, self.Y_test))
self.model.save_weights('MNISTWeights.h5', overwrite=True)
score = self.model.evaluate(self.X_test, self.Y_test, verbose=0)
print('Test score:', score[0])
print('Test accuracy:', score[1])
def predict(self, data):
data = data.reshape(data.shape[0], 1, 28, 28).astype('float32') / 255
return self.model.predict_classes(data, batch_size=32)
def create_neural_network(self):
model = Sequential()
model.add(Dense(100,input_dim=self.nstates, activation='relu'))
model.add(Dense(100,activation='relu'))
model.add(Dense(self.nactions,activation='linear'))
model.compile(loss='mse', optimizer=Adam(lr=self.alpha))
model_json = model.to_json()
with open('cartpole.json','w') as json_file:
json_file.write(model_json)
return model
def train(cwsInfo, cwsData, modelPath, weightPath):
(initProb, tranProb), (vocab, indexVocab) = cwsInfo
(X, y) = cwsData
train_X, test_X, train_y, test_y = train_test_split(X, y , train_size=0.9, random_state=1)
#train_X = [ [ [],[],...,[] ] ] train_y = [ [],[],...,[] ]
train_X = np.array(train_X)
train_y = np.array(train_y)
test_X = np.array(test_X)
test_y = np.array(test_y)
outputDims = len(corpus_tags)
Y_train = np_utils.to_categorical(train_y, outputDims)
Y_test = np_utils.to_categorical(test_y, outputDims)
batchSize = 128
vocabSize = len(vocab) + 1
wordDims = 100
maxlen = 7
hiddenDims = 100
w2vModel = Word2Vec.load('model/sougou.char.model')
embeddingDim = w2vModel.vector_size
embeddingUnknown = [0 for i in range(embeddingDim)]
embeddingWeights = np.zeros((vocabSize + 1, embeddingDim))
for word, index in vocab.items():
if word in w2vModel:
e = w2vModel[word]
else:
e = embeddingUnknown
embeddingWeights[index, :] = e
#LSTM
model = Sequential()
model.add(Embedding(output_dim = embeddingDim, input_dim = vocabSize + 1,
input_length = maxlen, mask_zero = True, weights = [embeddingWeights]))
model.add(LSTM(output_dim = hiddenDims, return_sequences = True))
model.add(LSTM(output_dim = hiddenDims, return_sequences = False))
model.add(Dropout(0.5))
model.add(Dense(outputDims))
model.add(Activation('softmax'))
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam')
result = model.fit(train_X, Y_train, batch_size = batchSize,
nb_epoch = 20, validation_data = (test_X,Y_test), show_accuracy=True)
j = model.to_json()
fd = open(modelPath, 'w')
fd.write(j)
fd.close()
model.save_weights(weightPath)
return model
def test_Bidirectional():
rnn = layers.SimpleRNN
samples = 2
dim = 2
timesteps = 2
output_dim = 2
dropout_rate = 0.2
for mode in ['sum', 'concat']:
x = np.random.random((samples, timesteps, dim))
target_dim = 2 * output_dim if mode == 'concat' else output_dim
y = np.random.random((samples, target_dim))
# test with Sequential model
model = Sequential()
model.add(wrappers.Bidirectional(rnn(output_dim, dropout=dropout_rate,
recurrent_dropout=dropout_rate),
merge_mode=mode,
input_shape=(timesteps, dim)))
model.compile(loss='mse', optimizer='sgd')
model.fit(x, y, epochs=1, batch_size=1)
# test config
model.get_config()
model = model_from_json(model.to_json())
model.summary()
# test stacked bidirectional layers
model = Sequential()
model.add(wrappers.Bidirectional(rnn(output_dim,
return_sequences=True),
merge_mode=mode,
input_shape=(timesteps, dim)))
model.add(wrappers.Bidirectional(rnn(output_dim), merge_mode=mode))
model.compile(loss='mse', optimizer='sgd')
model.fit(x, y, epochs=1, batch_size=1)
# test with functional API
inputs = Input((timesteps, dim))
outputs = wrappers.Bidirectional(rnn(output_dim, dropout=dropout_rate,
recurrent_dropout=dropout_rate),
merge_mode=mode)(inputs)
if dropout_rate and K.backend() != 'cntk':
# Dropout is disabled with CNTK for now.
assert outputs._uses_learning_phase
model = Model(inputs, outputs)
model.compile(loss='mse', optimizer='sgd')
model.fit(x, y, epochs=1, batch_size=1)
# Bidirectional and stateful
inputs = Input(batch_shape=(1, timesteps, dim))
outputs = wrappers.Bidirectional(rnn(output_dim, stateful=True),
merge_mode=mode)(inputs)
model = Model(inputs, outputs)
model.compile(loss='mse', optimizer='sgd')
model.fit(x, y, epochs=1, batch_size=1)
def initialize_model():
model = Sequential()
model.add(Conv2D(16, (4, 4), strides=(2, 2), padding='valid',
input_shape=(IMG_WIDTH, IMG_HEIGHT, 3)))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(16, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(32, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(50))
model.add(BatchNormalization())
model.add(Activation('relu'))
model.add(Dense(NB_CLASSES))
model.add(BatchNormalization())
model.add(Activation('softmax'))
opt = Adam(lr=LEARNING_RATE, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
model.compile(loss = "categorical_crossentropy", optimizer = opt, metrics=['accuracy'])
print model.summary()
model_json = model.to_json()
with open("model.json", "w") as json_file:
json_file.write(model_json)
return model
def cnn():
train_data, train_label, test_data, test_label = load_data()
train_data = train_data / 255.0
test_data = test_data / 255.0
print(train_data.shape[0], 'samples')
print(test_data.shape[0]), 'test'
print train_label
train_label = np_utils.to_categorical(train_label, 28) # divide into 28 categories
test_label = np_utils.to_categorical(test_label, 28)
model = Sequential()
model.add(Convolution2D(16, 4, 4,
border_mode='valid',
input_shape=(1, 28, 28)))
model.add(Activation('relu'))
model.add(Convolution2D(32, 4, 4, border_mode='full'))
model.add(Activation('relu'))
model.add(Convolution2D(32, 4, 4, border_mode='full'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Convolution2D(64, 4, 4, border_mode='full'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 4, 4, border_mode='full'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(28))
model.add(Activation('softmax'))
model.load_weights('model_weights.h5')
model.compile(loss='categorical_crossentropy', optimizer='adadelta')
model.fit(train_data, train_label, batch_size=100, nb_epoch=10, shuffle=True, verbose=1, show_accuracy=True, validation_data=(test_data, test_label))
json_string = model.to_json()
open('model_architecture.json','w').write(json_string)
model.save_weights('model_weights.h5')
def test_constant_initializer_with_numpy():
model = Sequential()
model.add(Dense(2, input_shape=(3,),
kernel_initializer=Constant(np.ones((3, 2)))))
model.add(Dense(3))
model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
json_str = model.to_json()
model_from_json(json_str).summary()
yaml_str = model.to_yaml()
model_from_yaml(yaml_str).summary()
def try_params( n_iterations, params, data=None, datamode='memory'):
print "iterations:", n_iterations
print_params( params )
batchsize = 100
if datamode == 'memory':
X_train, Y_train = data['train']
X_valid, Y_valid = data['valid']
inputshape = X_train.shape[1:]
else:
train_generator = data['train']['gen_func'](batchsize, data['train']['path'])
valid_generator = data['valid']['gen_func'](batchsize, data['valid']['path'])
train_epoch_step = data['train']['n_sample'] / batchsize
valid_epoch_step = data['valid']['n_sample'] / batchsize
inputshape = data['train']['gen_func'](batchsize, data['train']['path']).next()[0].shape[1:]
model = Sequential()
model.add(Conv2D(128, (1, 24), padding='same', input_shape=inputshape, activation='relu'))
model.add(GlobalMaxPooling2D())
model.add(Dense(32,activation='relu'))
model.add(Dropout(params['DROPOUT']))
model.add(Dense(2))
model.add(Activation('softmax'))
optim = Adadelta
myoptimizer = optim(epsilon=params['DELTA'], rho=params['MOMENT'])
mylossfunc = 'categorical_crossentropy'
model.compile(loss=mylossfunc, optimizer=myoptimizer,metrics=['accuracy'])
early_stopping = EarlyStopping( monitor = 'val_loss', patience = 3, verbose = 0 )
if datamode == 'memory':
model.fit(
X_train,
Y_train,
batch_size=batchsize,
epochs=int( round( n_iterations )),
validation_data=(X_valid, Y_valid),
callbacks = [ early_stopping ])
score, acc = model.evaluate(X_valid,Y_valid)
else:
model.fit_generator(
train_generator,
steps_per_epoch=train_epoch_step,
epochs=int( round( n_iterations )),
validation_data=valid_generator,
validation_steps=valid_epoch_step,
callbacks = [ early_stopping ])
score, acc = model.evaluate_generator(valid_generator, steps=valid_epoch_step)
return { 'loss': score, 'model': (model.to_json(), optim, myoptimizer.get_config(), mylossfunc) }
class Toy01:
"""Sets up and trains the ANN using keras"""
def __init__(self, xtrain, ytrain, results_dir):
self.xtrain = xtrain
self.ytrain = ytrain
self.results_dir = results_dir
# ANN model
self.optimizer = optimizers.RMSprop(lr=0.005)
self.loss = 'mse'
self.metrics = 'accuracy'
self.model = Sequential()
self.model.add(Dense(output_dim=30, input_dim=2, init='uniform'))
self.model.add(Activation('relu'))
self.model.add(Dense(output_dim=30, init='uniform'))
self.model.add(Activation('relu'))
self.model.add(Dense(output_dim=20, init='uniform'))
self.model.add(Activation('relu'))
self.model.add(Dense(output_dim=1, init='uniform'))
self.model.add(Activation('sigmoid'))
# Compile the model
self.model.compile(
loss=self.loss,
optimizer=self.optimizer,
metrics=[self.metrics]
)
def process(self, nepochs):
progress_fname = os.path.join(self.results_dir, 'progress.txt')
model_fname = os.path.join(self.results_dir, 'ann_model.txt')
weights_dir = os.path.join(self.results_dir, 'weights')
if not os.path.exists(weights_dir):
os.mkdir(weights_dir, 0o755)
step = EpochInstrospector(self.model, self.optimizer, progress_fname, weights_dir)
fout = open(progress_fname, 'w')
fout.close()
with open(model_fname, 'w') as fout:
jmodel = self.model.to_json()
fout.write(jmodel)
self.model.fit(
self.xtrain,
self.ytrain,
batch_size=30,
nb_epoch=nepochs,
verbose=1,
callbacks=[step],
validation_split=0.0,
validation_data=None,
shuffle=True,
class_weight=None,
sample_weight=None
)
def test_TimeDistributed():
# first, test with Dense layer
model = Sequential()
model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 2)), nb_epoch=1, batch_size=10)
# test config
model.get_config()
# compare to TimeDistributedDense
test_input = np.random.random((1, 3, 4))
test_output = model.predict(test_input)
weights = model.layers[0].get_weights()
reference = Sequential()
reference.add(core.TimeDistributedDense(2, input_shape=(3, 4), weights=weights))
reference.add(core.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test when specifying a batch_input_shape
reference = Sequential()
reference.add(core.TimeDistributedDense(2, batch_input_shape=(1, 3, 4), weights=weights))
reference.add(core.Activation('relu'))
reference.compile(optimizer='rmsprop', loss='mse')
reference_output = reference.predict(test_input)
assert_allclose(test_output, reference_output, atol=1e-05)
# test with Convolution2D
model = Sequential()
model.add(wrappers.TimeDistributed(convolutional.Convolution2D(5, 2, 2, border_mode='same'), input_shape=(2, 3, 4, 4)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.train_on_batch(np.random.random((1, 2, 3, 4, 4)), np.random.random((1, 2, 5, 4, 4)))
model = model_from_json(model.to_json())
model.summary()
# test stacked layers
model = Sequential()
model.add(wrappers.TimeDistributed(core.Dense(2), input_shape=(3, 4)))
model.add(wrappers.TimeDistributed(core.Dense(3)))
model.add(core.Activation('relu'))
model.compile(optimizer='rmsprop', loss='mse')
model.fit(np.random.random((10, 3, 4)), np.random.random((10, 3, 3)), nb_epoch=1, batch_size=10)
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <data directory>\n")
sys.exit(-1)
working_dir = args[0]
print("Reading data from %s..." % working_dir)
Y, X = ctk_io.read_liblinear(working_dir)
num_outputs = Y.shape[-1]
num_examples, dimension = X.shape
num_y_examples, num_labels = Y.shape
assert num_examples == num_y_examples
model = Sequential()
sgd = SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True)
# Dense(10) is a fully-connected layer with 10 hidden units.
# in the first layer, you must specify the expected input data shape:
# here, 20-dimensional vectors.
model.add(Dense(10, input_dim=dimension, init='uniform'))
model.add(Activation('relu'))
model.add(Dense(10, init='uniform'))
model.add(Activation('relu'))
if num_outputs > 1:
model.add(Dense(num_outputs, init='uniform'))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
else:
model.add(Dense(1, init='uniform'))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
print("Shape of y is %s, shape of X is %s, max value in y is %f and min is %f with %d outcomes" % (str(Y.shape), str(X.shape), Y.max(), Y.min(), num_outputs) )
model.fit(X, Y,
nb_epoch=nb_epoch,
batch_size=batch_size,
verbose=1)
model.summary()
json_string = model.to_json()
open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string)
model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True)
def train_xy(epochs=50, batch_size=32, h=256, w=256, ch=3, train_p=0.8, valid_p=0.1):
print("Compiling Model")
t_comp = time()
model = Sequential()
# reshape input to ch, h, w (no sample axis)
model.add(Reshape(dims=(h, w, ch), input_shape=(ch * h * w,)))
model.add(Permute((3, 1, 2)))
# add conv layers
model.add(Convolution2D(16, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(32, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, init="glorot_uniform", activation="relu", subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(output_dim=2000, init="glorot_uniform", activation="relu", W_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(output_dim=2000, init="glorot_uniform", activation="relu", W_regularizer=l2(0.001)))
model.add(Dropout(0.5))
model.add(Dense(output_dim=4, init="glorot_uniform", activation="relu"))
model.compile(optimizer="rmsprop", loss="mse")
t_train = time()
print("Took %.1fs" % (t_train - t_comp))
# split dataset
i_test = int(train_p * nrow) / batch_size * batch_size
i_valid = int(i_test * (1 - valid_p)) / batch_size * batch_size
X_train, Y_train = X_[:i_valid,], Y_[:i_valid,]
X_valid, Y_valid = X_[i_valid:i_test,], Y_[i_valid:i_test,]
# naive fitting to lower rmse faster
hist = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=10, verbose=1, validation_split=0.1)
print(hist)
# fit by batch using generator!
img_aug = image_augmentor(X_, Y_, i_valid)
hist = model.fit_generator(
generator=img_aug,
samples_per_epoch=i_valid,
nb_epoch=5000,
verbose=1,
validation_data=(X_valid, Y_valid),
nb_worker=1,
)
rmse_test = model.evaluate(X_[i_test:,], Y_[i_test:,])
print("Test RMSE: %.4f" % rmse_test)
# save model
model_json = model.to_json()
open(path_img + "locate/model_116.json", "w").write(model_json)
model.save_weights(path_img + "locate/model_116_weights.h5")
def buildModel():
# build the model: 2 stacked LSTM
print('Build model...')
model = Sequential()
# model.add(Dense(512, input_dim= maxlen * len(chars), init='uniform'))
# model.add(Activation('relu'))
# model.add(Dropout(0.1))
'''
model.add(Dense(256, input_dim= maxlen * len(chars), init='uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(Dense(128, input_dim= maxlen * len(chars), init='uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.1))
model.add(Dense(64, input_dim= maxlen * len(chars), init='uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.1))
'''
# model.add(LSTM(9, return_sequences=False, input_shape=(maxlen, len(chars))))
model.add(LSTM(32, return_sequences=True, input_shape=(maxlen, len(chars))))
model.add(Dropout(0.1))
model.add(LSTM(32, return_sequences=False))
model.add(Dropout(0.1))
model.add(Dense(32, init='uniform'))
model.add(Activation('relu'))
model.add(Dropout(0.1))
#model.add(LSTM(32, return_sequences=True))
#model.add(Dropout(0.1))
#model.add(LSTM(32, return_sequences=False))
#model.add(Dropout(.1))
model.add(Dense(len(chars)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
# Save model
json_string = model.to_json()
model_file = open(output_folder + "model.json", "w+")
model_file.write(json_string)
model_file.close()
return model
