def build_model(layers):
model = Sequential()
model.add(LSTM(
input_shape = (None,layers[0]),
units=layers[1],
return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(
input_shape = (layers[1], 1024),
units=1024,
return_sequences=True))
model.add(Dropout(0.4))
model.add(LSTM(
1024,
return_sequences=False))
model.add(Dropout(0.4))
model.add(Dense(
units=layers[3]))
model.add(Activation("linear"))
start = time.time()
model.compile(loss="mse", optimizer="rmsprop",metrics=['accuracy'])
print("> Compilation Time : ", time.time() - start)
return model
def lstm(trainData, trainMark, testData, embedding_dim, embedding_matrix, maxlen, output_len):
# 填充数据，将每个序列长度保持一致
trainData = list(sequence.pad_sequences(trainData, maxlen=maxlen,
dtype='float64')) # sequence返回的是一个numpy数组，pad_sequences用于填充指定长度的序列，长则阶段，短则补0，由于下面序号为0时，对应值也为0，因此可以这样
testData = list(sequence.pad_sequences(testData, maxlen=maxlen,
dtype='float64')) # sequence返回的是一个numpy数组，pad_sequences用于填充指定长度的序列，长则阶段，短则补0
# 建立lstm神经网络模型
model = Sequential() # 多个网络层的线性堆叠，可以通过传递一个layer的list来构造该模型，也可以通过.add()方法一个个的加上层
# model.add(Dense(256, input_shape=(train_total_vova_len,))) #使用全连接的输入层
model.add(Embedding(len(embedding_matrix), embedding_dim, weights=[embedding_matrix], mask_zero=False,
input_length=maxlen)) # 指定输入层，将高维的one-hot转成低维的embedding表示，第一个参数大或等于0的整数，输入数据最大下标+1，第二个参数大于0的整数，代表全连接嵌入的维度
# lstm层，也是比较核心的层
model.add(LSTM(256)) # 256对应Embedding输出维度，128是输入维度可以推导出来
model.add(Dropout(0.5)) # 每次在参数更新的时候以一定的几率断开层的链接，用于防止过拟合
model.add(Dense(output_len)) # 全连接，这里用于输出层，1代表输出层维度，128代表LSTM层维度可以自行推导出来
model.add(Activation('softmax')) # 输出用sigmoid激活函数
# 编译该模型，categorical_crossentropy（亦称作对数损失，logloss），adam是一种优化器，class_mode表示分类模式
model.compile(loss='categorical_crossentropy', optimizer='sgd')
# 正式运行该模型,我知道为什么了，因为没有补0！！每个array的长度是不一样的，因此才会报错
X = np.array(list(trainData)) # 输入数据
print("X:", X)
Y = np.array(list(trainMark)) # 标签
print("Y:", Y)
# batch_size：整数，指定进行梯度下降时每个batch包含的样本数
# nb_epoch：整数，训练的轮数，训练数据将会被遍历nb_epoch次
model.fit(X, Y, batch_size=200, nb_epoch=10) # 该函数的X、Y应该是多个输入：numpy list(其中每个元素为numpy.array)，单个输入：numpy.array
# 进行预测
A = np.array(list(testData)) # 输入数据
print("A:", A)
classes = model.predict(A) # 这个是预测的数据
return classes
def deep_CNN(input_shape, label_class):
#Two Convolutional Layers with Pooling Layer
#config
nb_filter = [250, 150, 50]
nb_row = [1, 1]
nb_col = [1, 1]
nb_pool_size = [1, 1]
#init
model = Sequential()
#First layers
model.add(Convolution2D( nb_filter = 27, nb_row = 3, nb_col = 1,
border_mode = 'valid', activation = 'relu',
input_shape=(input_shape[0],input_shape[1],input_shape[2])
))
model.add(Convolution2D( nb_filter = 48, nb_row = 3, nb_col = 300,
border_mode = 'valid', activation = 'relu'
))
#pooling layer
model.add(MaxPooling2D( pool_size=(21,1) ))
#Fully Connected Layer with dropout
model.add(Flatten())
model.add(Dense(output_dim = 256, activation = 'relu'))
model.add(Dropout(0.5))
#Fully Connected Layer as output layer
model.add(Dense(output_dim = label_class, activation='sigmoid'));
adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
model.compile(loss='binary_crossentropy', class_mode = 'categorical',
optimizer = adadelta)
return model
def train():
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 128, input_length=maxlen, dropout=0.2))
model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2)) # try using a GRU instead, for fun
model.add(Dense(1))
model.add(Activation('sigmoid'))
# try using different optimizers and different optimizer configs
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
print(X_train.shape)
print(y_train.shape)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=15,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test,
batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
with open("save_weight_lstm.pickle", mode="wb") as f:
pickle.dump(model.get_weights(),f)
def parallel_CNN():
filter_shapes = [ [2, 300], [3, 300], [4, 300], [5, 300] ]
pool_shapes = [ [25, 1], [24, 1], [23, 1], [22, 1] ]
#Four Parallel Convolutional Layers with Four Pooling Layers
model = Sequential()
sub_models = []
for i in range( len(pool_shapes) ):
pool_shape = pool_shapes[i]
filter_shape = filter_shapes[i]
sub_model = Sequential()
sub_model.add( Convolution2D(nb_filter = 512, nb_row = filter_shape[0], nb_col = filter_shape[1],
border_mode='valid', activation='relu',
input_shape=(input_shape[0], input_shape[1], input_shape[2])
))
sub_model.add( MaxPooling2D(pool_size=(pool_shape[1], pool_shape[1])) )
sub_models.append(sub_model)
model.add(Merge(sub_models, mode='concat'))
#Fully Connected Layer with dropout
model.add(Flatten())
model.add(Dense(output_dim=256, activation='relu', input_dim=2048))
model.add(Dropout(0.5))
#Fully Connected Layer as output layer
model.add( Dense(output_dim=label_num, activation='sigmoid', input_dim=256))
adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-6)
model.compile(loss='binary_crossentropy', class_mode = 'multi_label',
optimizer=adadelta)
return model
def train_model():
# (X_train, Y_train, X_test, Y_test) = prapare_train()
X_ = []
with open('../data/train_matrix.out') as train_file:
X_train = json.load(train_file)
for x in X_train:
a = len(x)
print a/2
x1 = x[:a/2]
x2 = x[a/2:]
x3 = []
x3.append(x1)
x3.append(x2)
X_.append(x3)
# X_test = pickle.load('../data/test_matrix.out')
Y_train = [1,0,0]*3
# Y_test = [1,0,0]*3
# print len(X_train) - len(Y_train)
# print len(X_test) - len(Y_test)
model = Sequential()
model = get_nn_model()
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# model.fit(X_train, Y_train,
# batch_size=batch_size,
# nb_epoch=nb_epoch,
# validation_data=(X_test, Y_test))
#2
model.fit(X_, Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_split = 0.2)
print 'ok'
def model(X_train, X_test, y_train, y_test, maxlen, max_features):
embedding_size = 300
pool_length = 4
lstm_output_size = 100
batch_size = 200
nb_epoch = 1
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout({{uniform(0, 1)}}))
# Note that we use unnamed parameters here, which is bad style, but is used here
# to demonstrate that it works. Always prefer named parameters.
model.add(Convolution1D({{choice([64, 128])}},
{{choice([6, 8])}},
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def test_sequential_model_saving():
model = Sequential()
model.add(Dense(2, input_shape=(3,)))
model.add(RepeatVector(3))
model.add(TimeDistributed(Dense(3)))
model.compile(loss=losses.MSE,
optimizer=optimizers.RMSprop(lr=0.0001),
metrics=[metrics.categorical_accuracy],
sample_weight_mode='temporal')
x = np.random.random((1, 3))
y = np.random.random((1, 3, 3))
model.train_on_batch(x, y)
out = model.predict(x)
_, fname = tempfile.mkstemp('.h5')
save_model(model, fname)
new_model = load_model(fname)
os.remove(fname)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
# test that new updates are the same with both models
x = np.random.random((1, 3))
y = np.random.random((1, 3, 3))
model.train_on_batch(x, y)
new_model.train_on_batch(x, y)
out = model.predict(x)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
def cnn():
model = Sequential()
# **Worth taking into consideration that our image size is tiny (8x8), convolution may work much better for
# **with 1792 sipms
# kernal size is 3x3, 32 filters, padding is same.
# Same padding works better, this is probably because same padding makes it easier for network no to retain as
# much information as possible around the edges.
model.add(Convolution2D(32,3,3,border_mode='same',input_shape=(1, nsipm, nsipm)))
model.add(Activation('relu'))
model.add(Convolution2D(32,3,3,border_mode='same', input_shape=(32, nsipm, nsipm)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(output_dim=128))
model.add(Activation('relu'))
model.add(Dense(output_dim=64))
model.add(Activation('relu'))
model.add(Dense(output_dim=2*N_ELpts))
model.add(Activation('sigmoid'))
# Nadam optimizer is a safe choice at least for deep networks. It is adam optimizer with Nesterov
# Momentum. Nesterov Momentum takes into account future expected future gradient gradient, unlike traditional Mom.
model.compile(loss='mse', optimizer=Nadam(lr=0.002, beta_1=0.9, beta_2=0.999, epsilon=1e-08, schedule_decay=0.004))
N_layers = 'cnn'
return model,N_layers
def test_TensorBoard_with_ReduceLROnPlateau(tmpdir):
import shutil
np.random.seed(np.random.randint(1, 1e7))
filepath = str(tmpdir / 'logs')
(X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim,),
classification=True,
num_classes=num_class)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
model = Sequential()
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_class, activation='softmax'))
model.compile(loss='binary_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
cbks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.5,
patience=4,
verbose=1),
callbacks.TensorBoard(
log_dir=filepath)]
model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=cbks, epochs=2)
assert os.path.isdir(filepath)
shutil.rmtree(filepath)
assert not tmpdir.listdir()
def model_1(lr=.001, rho=.9, epsilon=1.0e-6):
dnn = Sequential()
dnn.add(BatchNormalization(input_shape=(3, 101, 101)))
dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
dnn.add(MaxPooling2D())
dnn.add(LeakyReLU(alpha=.01))
dnn.add(Convolution2D(16, 3, 3, init='he_normal'))
dnn.add(MaxPooling2D())
dnn.add(LeakyReLU(alpha=.01))
dnn.add(Convolution2D(16, 3, 3, init='he_normal'))
dnn.add(MaxPooling2D())
dnn.add(LeakyReLU(alpha=.01))
dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
dnn.add(MaxPooling2D())
dnn.add(LeakyReLU(alpha=.01))
dnn.add(Convolution2D(16, 2, 2, init='he_normal'))
dnn.add(MaxPooling2D())
dnn.add(LeakyReLU(alpha=.01))
dnn.add(Flatten())
dnn.add(Dense(100))
dnn.add(Dense(2))
dnn.add(Activation('softmax'))
dnn.compile(loss='binary_crossentropy', optimizer=Adamax(lr=lr))
return dnn
def test_EarlyStopping():
np.random.seed(1337)
(X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim,),
classification=True,
num_classes=num_class)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
model = Sequential()
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_class, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
mode = 'max'
monitor = 'val_acc'
patience = 0
cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
history = model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=cbks, epochs=20)
mode = 'auto'
monitor = 'val_acc'
patience = 2
cbks = [callbacks.EarlyStopping(patience=patience, monitor=monitor, mode=mode)]
history = model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=cbks, epochs=20)
def test_LambdaCallback():
np.random.seed(1337)
(X_train, y_train), (X_test, y_test) = get_test_data(num_train=train_samples,
num_test=test_samples,
input_shape=(input_dim,),
classification=True,
num_classes=num_class)
y_test = np_utils.to_categorical(y_test)
y_train = np_utils.to_categorical(y_train)
model = Sequential()
model.add(Dense(num_hidden, input_dim=input_dim, activation='relu'))
model.add(Dense(num_class, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# Start an arbitrary process that should run during model training and be terminated after training has completed.
def f():
while True:
pass
p = multiprocessing.Process(target=f)
p.start()
cleanup_callback = callbacks.LambdaCallback(on_train_end=lambda logs: p.terminate())
cbks = [cleanup_callback]
model.fit(X_train, y_train, batch_size=batch_size,
validation_data=(X_test, y_test), callbacks=cbks, epochs=5)
p.join()
assert not p.is_alive()
def ann(self):
#print self.company.X_train.shape[1]
model = Sequential()
model.add(Dense(input_dim=self.search_inputs.X_train.shape[1], output_dim=10, init="glorot_uniform"))
model.add(Activation('tanh'))
model.add(Dropout(0.1))
model.add(Dense(input_dim=10, output_dim=10, init="uniform"))
model.add(Activation('tanh'))
model.add(Dropout(0.5))
model.add(Dense(input_dim=10, output_dim=1, init="glorot_uniform"))
model.add(Activation("linear"))
sgd = SGD(lr=0.3, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='mean_squared_error', optimizer='sgd')
early_stopping = EarlyStopping(monitor='val_loss', patience=25)
#epoch_score = model.evaluate(X_score, y_score, batch_size = 16) # this doesn't work
# first model
print "fitting first model"
model.fit(self.search_inputs.X_train, self.search_inputs.y_train, nb_epoch=100, validation_split=.1, batch_size=100, verbose = 1, show_accuracy = True, shuffle = True, callbacks=[early_stopping])
#score = model.evaluate(self.company.X_cv, self.company.y_cv, show_accuracy=True, batch_size=16)
self.ann_preds = model.predict(self.search_inputs.X_test)
#just in case (like w/ svr)
for i in range(0,len(self.ann_preds) - 1):
if self.ann_preds[i] < 1:
self.ann_preds[i] = 1.00
elif self.ann_preds[i] > 3:
self.ann_preds[i] = 3.00
self.search_inputs.fin_df['relevance'] = np.array(self.ann_preds) # easy swap in / out
final_file_ann = self.search_inputs.fin_df.to_csv(self.fin_file_name+'_ann.csv', float_format='%.5f', index=False)
def define_model(lr, momentum):
# CONFIG
model = Sequential()
# Create Layers
# CONVNET
layers = []
#layers.append(GaussianNoise(0.02))
layers.append(Convolution2D(8, 9, 9, activation = "relu", input_shape=(1,100,100)))
layers.append(MaxPooling2D(pool_size=(2,2)))
layers.append(Convolution2D(16, 7, 7, activation = "relu"))
layers.append(MaxPooling2D(pool_size=(2,2)))
layers.append(Convolution2D(32, 5, 5, activation = "relu"))
layers.append(MaxPooling2D(pool_size=(2,2)))
layers.append(Convolution2D(64, 3, 3, activation = "relu"))
layers.append(MaxPooling2D(pool_size=(2,2)))
layers.append(Convolution2D(250, 3, 3, activation= "relu"))
# MLP
layers.append(Flatten())
layers.append(Dense(125, activation="relu"))
layers.append(Dense(2, activation="softmax"))
# Adding Layers
for layer in layers:
model.add(layer)
# COMPILE (learning rate, momentum, objective...)
sgd = SGD(lr=lr, momentum=momentum)
model.compile(loss="categorical_crossentropy", optimizer=sgd)
return model
def test_merge_overlap():
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, left], mode='sum'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_test, y_test))
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=2, validation_data=(X_test, y_test))
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_split=0.1)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=1, validation_split=0.1)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=1, shuffle=False)
model.train_on_batch(X_train[:32], y_train[:32])
loss = model.evaluate(X_train, y_train, verbose=0)
assert(loss < 0.7)
model.predict(X_test, verbose=0)
model.predict_classes(X_test, verbose=0)
model.predict_proba(X_test, verbose=0)
model.get_config(verbose=0)
fname = 'test_merge_overlap_temp.h5'
model.save_weights(fname, overwrite=True)
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate(X_train, y_train, verbose=0)
assert(loss == nloss)
class QLearn:
def __init__(self, actions, epsilon, alpha, gamma):
# instead of a dictionary, we'll be using
# a neural network
# self.q = {}
self.epsilon = epsilon # exploration constant
self.alpha = alpha # discount constant
self.gamma = gamma # discount factor
self.actions = actions
# Build the neural network
self.network = Sequential()
self.network.add(Dense(50, init='lecun_uniform', input_shape=(4,)))
# self.network.add(Activation('sigmoid'))
#self.network.add(Dropout(0.2))
self.network.add(Dense(20, init='lecun_uniform'))
# self.network.add(Activation('sigmoid'))
# #self.network.add(Dropout(0.2))
self.network.add(Dense(2, init='lecun_uniform'))
# self.network.add(Activation('linear')) #linear output so we can have range of real-valued outputs
# rms = SGD(lr=0.0001, decay=1e-6, momentum=0.5) # explodes to non
rms = RMSprop()
# rms = Adagrad()
# rms = Adam()
self.network.compile(loss='mse', optimizer=rms)
# Get a summary of the network
self.network.summary()
def train_rnn(character_corpus, seq_len, train_test_split_ratio):
model = Sequential()
model.add(Embedding(character_corpus.char_num(), 256))
model.add(LSTM(256, 5120, activation='sigmoid', inner_activation='hard_sigmoid', return_sequences=True))
model.add(Dropout(0.5))
model.add(TimeDistributedDense(5120, character_corpus.char_num()))
model.add(Activation('time_distributed_softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
seq_X, seq_Y = character_corpus.make_sequences(seq_len)
print "Sequences are made"
train_seq_num = train_test_split_ratio*seq_X.shape[0]
X_train = seq_X[:train_seq_num]
Y_train = to_time_distributed_categorical(seq_Y[:train_seq_num], character_corpus.char_num())
X_test = seq_X[train_seq_num:]
Y_test = to_time_distributed_categorical(seq_Y[train_seq_num:], character_corpus.char_num())
print "Begin train model"
checkpointer = ModelCheckpoint(filepath="model.step", verbose=1, save_best_only=True)
model.fit(X_train, Y_train, batch_size=256, nb_epoch=100, verbose=2, validation_data=(X_test, Y_test), callbacks=[checkpointer])
print "Model is trained"
score = model.evaluate(X_test, Y_test, batch_size=512)
print "valid score = ", score
return model
def test_merge_dot():
if K._BACKEND == 'tensorflow':
return
left = Sequential()
left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=1))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
left = Sequential()
left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=([1], [1])))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
def test_sequential_model_saving():
model = Sequential()
model.add(Dense(2, input_dim=3))
model.add(Dense(3))
model.compile(loss='mse', optimizer='rmsprop', metrics=['acc'])
x = np.random.random((1, 3))
y = np.random.random((1, 3))
model.train_on_batch(x, y)
out = model.predict(x)
fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
save_model(model, fname)
new_model = load_model(fname)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
# test that new updates are the same with both models
x = np.random.random((1, 3))
y = np.random.random((1, 3))
model.train_on_batch(x, y)
new_model.train_on_batch(x, y)
out = model.predict(x)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
# test load_weights on model file
model.load_weights(fname)
os.remove(fname)
def test_sequential_model_saving_2():
# test with funkier config
model = Sequential()
model.add(Dense(2, input_dim=3))
model.add(RepeatVector(3))
model.add(TimeDistributed(Dense(3)))
model.compile(loss=objectives.MSE,
optimizer=optimizers.RMSprop(lr=0.0001),
metrics=[metrics.categorical_accuracy],
sample_weight_mode='temporal')
x = np.random.random((1, 3))
y = np.random.random((1, 3, 3))
model.train_on_batch(x, y)
out = model.predict(x)
fname = 'tmp_' + str(np.random.randint(10000)) + '.h5'
save_model(model, fname)
new_model = load_model(fname)
os.remove(fname)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
# test that new updates are the same with both models
x = np.random.random((1, 3))
y = np.random.random((1, 3, 3))
model.train_on_batch(x, y)
new_model.train_on_batch(x, y)
out = model.predict(x)
out2 = new_model.predict(x)
assert_allclose(out, out2, atol=1e-05)
def model(X_train, Y_train, X_test, Y_test):
'''
Model providing function:
Create Keras model with double curly brackets dropped-in as needed.
Return value has to be a valid python dictionary with two customary keys:
- loss: Specify a numeric evaluation metric to be minimized
- status: Just use STATUS_OK and see hyperopt documentation if not feasible
The last one is optional, though recommended, namely:
- model: specify the model just created so that we can later use it again.
'''
model = Sequential()
model.add(Dense(512, input_shape=(784,)))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense({{choice([256, 512, 1024])}}))
model.add(Activation('relu'))
model.add(Dropout({{uniform(0, 1)}}))
model.add(Dense(10))
model.add(Activation('softmax'))
rms = RMSprop()
model.compile(loss='categorical_crossentropy', optimizer=rms, metrics=['accuracy'])
model.fit(X_train, Y_train,
batch_size={{choice([64, 128])}},
nb_epoch=1,
verbose=2,
validation_data=(X_test, Y_test))
score, acc = model.evaluate(X_test, Y_test, verbose=0)
print('Test accuracy:', acc)
return {'loss': -acc, 'status': STATUS_OK, 'model': model}
def __init__(self, nb_filters=32, nb_conv=3, nb_pool=2):
model = Sequential()
model.add(Convolution2D(nb_filters, nb_conv, nb_conv,
border_mode='valid',
input_shape=(1, 5, 30)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Convolution2D(nb_filters/2, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.2))
model.add(Flatten())
model.add(Dense(500))
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(500))
model.add(Dropout(0.2))
model.add(Activation('relu'))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
self.model = model
def simple_cnn_vgg_like(lr=1e-3, weights_path=None):
img_rows, img_cols = 210, 70
# standard VGG16 network architecture
structure_path = "%s/cache/simple_cnn_vgg_like.json" % config.project.project_path
if weights_path is not None and os.path.exists(weights_path) \
and os.path.exists(structure_path):
logger.debug("load weigth from fine-tuning weight %s" % weights_path)
model = model_from_json(open(structure_path).read())
model.load_weights(weights_path)
else:
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(1, img_rows, img_cols)))
model.add(Convolution2D(64, 7, 7, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(64, 7, 7, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(Flatten())
model.add(Dense(4096, activation='relu'))
model.add(Dropout(0.5))
# replace more fc layer
model.add(Dense(124, activation='softmax'))
# load the weights
logger.debug('Model loaded.')
sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
return model
def simple_cnn_for_test(lr=1e-3, weights_path=None):
img_rows, img_cols = 210, 70
if weights_path is not None and os.path.exists(weights_path):
logging.debug("load weigth from fine-tuning weight %s" % weights_path)
model = load_model(weights_path)
else:
model = Sequential()
model.add(ZeroPadding2D((1, 1), input_shape=(1, img_rows, img_cols)))
model.add(Convolution2D(6, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(6, 3, 3, activation='relu'))
model.add(MaxPooling2D((2, 2), strides=(2, 2)))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(6, 3, 3, activation='relu'))
model.add(ZeroPadding2D((1, 1)))
model.add(Convolution2D(6, 3, 3, activation='relu'))
model.add(Flatten())
# replace more fc layer
model.add(Dense(124, activation='softmax'))
# load the weights
logging.debug('Model loaded.')
sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
return model
def baseline_model(): model = Sequential() model.add(Dense(4, input_dim=4, init='normal', activation='relu')) model.add(Dense(3, init='normal', activation='sigmoid')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def get_ts_model( trainX, trainY, look_back = 1, nb_epochs = 100 ):
model = Sequential()
# takes input array of shape (*, 1) where (2,1) - (row,col) array example looks like [23]
# [43]
model.add(LSTM(20, input_shape=(None , look_back) ))
#model.add(LSTM(20, batch_input_shape=(None, None, look_back), return_sequences= True ))
#print(model.summary)
model.add( Dense(1) )
model.add(Dense(1))
model.add(Dense(1))
model.add(Dense(1))
model.add(Dense(1))
model.add(Dense(1))
#model.add(LSTM(1, return_sequences= True))
#model.add(LSTM(1))
# outputs array of shape (*,1)
#model.add(Dense(1))
#model.compile(loss='mean_absolute_error', optimizer='SGD') # mape
#model.compile(loss='poisson', optimizer='adam') # mape
model.compile( loss = 'mean_squared_error', optimizer = 'adam' ) # values closer to zero are better.
#model.compile(loss='mean_squared_error', optimizer='adagrad')
# Values of MSE are used for comparative purposes of two or more statistical meythods. Heavily weight outliers, i.e weighs large errors more heavily than the small ones.
# "In cases where this is undesired, mean absolute error is used.
# REF: Available loss functions https://keras.io/objectives.
print('Start : Training model')
# default configuration
model.fit(trainX, trainY, nb_epoch=nb_epochs, batch_size=1, verbose=2)
#model.fit(trainX, trainY, nb_epoch=100, batch_size=1, verbose=2)
print('Ends : Training Model')
return model
def create_model(x_train, y_train, x_test, y_test):
"""
Create your model...
"""
layer_1_size = {{quniform(12, 256, 4)}}
l1_dropout = {{uniform(0.001, 0.7)}}
params = {
'l1_size': layer_1_size,
'l1_dropout': l1_dropout
}
num_classes = 10
model = Sequential()
model.add(Dense(int(layer_1_size), activation='relu'))
model.add(Dropout(l1_dropout))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=128, epochs=10, validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test, verbose=0)
out = {
'loss': -acc,
'score': score,
'status': STATUS_OK,
'model_params': params,
}
# optionally store a dump of your model here so you can get it from the database later
temp_name = tempfile.gettempdir()+'/'+next(tempfile._get_candidate_names()) + '.h5'
model.save(temp_name)
with open(temp_name, 'rb') as infile:
model_bytes = infile.read()
out['model_serial'] = model_bytes
return out
def _small_model(self):
'''
Alternative model architecture with fewer layers for computationally expensive
training datasets
'''
print 'Compiling Small Net...'
model = Sequential()
model.add(ZeroPadding2D((1,1), input_shape=self.input_shape))
model.add(Convolution2D(64, self.kernel_size, self.kernel_size,activation='relu',
input_shape=self.input_shape))
model.add(ZeroPadding2D((1,1)))
model.add(Convolution2D(64, self.kernel_size, self.kernel_size,
activation='relu'))
model.add(MaxPooling2D((2,2), strides=(2,2)))
model.add(ZeroPadding2D((1,1)))
model.add(Convolution2D(128, self.kernel_size, self.kernel_size,
activation='relu'))
model.add(ZeroPadding2D((1,1)))
model.add(Convolution2D(128, self.kernel_size, self.kernel_size,
activation='relu'))
model.add(MaxPooling2D((2,2), strides=(2,2)))
model.add(Flatten())
model.add(Dense(2048, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2048, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(self.nb_classes, activation='softmax'))
sgd = SGD(lr=self.lr, decay=0.01, momentum=0.9, nesterov=True)
model.compile(optimizer = 'sgd', loss = 'categorical_crossentropy')
return model
def compile_model(self):
'''
compiles standard single model with 4 convolitional/max-pooling layers.
'''
print 'Compiling single model...'
single = Sequential()
single.add(Convolution2D(self.n_filters[0], self.k_dims[0], self.k_dims[0], border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg), input_shape=(self.n_chan,33,33)))
single.add(Activation(self.activation))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[1], self.k_dims[1], self.k_dims[1], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[2], self.k_dims[2], self.k_dims[2], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(BatchNormalization(mode=0, axis=1))
single.add(MaxPooling2D(pool_size=(2,2), strides=(1,1)))
single.add(Dropout(0.5))
single.add(Convolution2D(self.n_filters[3], self.k_dims[3], self.k_dims[3], activation=self.activation, border_mode='valid', W_regularizer=l1l2(l1=self.w_reg, l2=self.w_reg)))
single.add(Dropout(0.25))
single.add(Flatten())
single.add(Dense(5))
single.add(Activation('softmax'))
sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
single.compile(loss='categorical_crossentropy', optimizer='sgd')
print 'Done.'
return single
class LanguageModel(RNNModel):
def __init__(self, *args, **kwargs):
'''
field is 'sentence' by default, but can be e.g. pos_sentence.
'''
self.field = kwargs.pop('field', 'sentence')
super(LanguageModel, self).__init__(*args, **kwargs)
self.class_to_code = {'VBZ': 0, 'VBP': 1}
self.inflect_verb, _ = gen_inflect_from_vocab(self.vocab_file)
def process_single_dependency(self, dep):
dep['label'] = dep['verb_pos']
tokens = dep[self.field].split()
return tokens
def create_train_and_test(self, examples):
random.seed(1)
random.shuffle(examples)
first = 1
self.X_train = []
self.Y_train = []
self.X_test = self.Y_test = [] # not used; just for compatibility
self.deps_train = []
n_train = int(len(examples) * self.prop_train)
for _, ints, dep in examples[:n_train]:
self.deps_train.append(dep)
for i in range(first, len(ints) - 1):
self.X_train.append(ints[:i])
self.Y_train.append(ints[i])
self.Y_train = np.asarray(self.Y_train)
self.deps_test = [x[2] for x in examples[n_train:]]
def create_model(self):
self.log('Creating model')
self.model = Sequential()
self.model.add(
Embedding(len(self.vocab_to_ints) + 1,
self.embedding_size,
input_length=self.maxlen))
self.model.add(
self.rnn_class(output_dim=self.rnn_output_size,
input_length=self.maxlen))
self.model.add(Dense(len(self.vocab_to_ints) + 1))
self.model.add(Activation('softmax'))
def compile_model(self):
self.log('Compiling model')
self.model.compile(loss='sparse_categorical_crossentropy',
optimizer='adam')
def results(self):
recs = []
columns = ['gram_loss', 'ungram_loss', 'correct'] + dependency_fields
self.model.model._make_test_function()
progbar = Progbar(len(self.deps_test))
for i, dep in enumerate(self.deps_test):
inp = np.zeros((1, self.maxlen))
v = int(dep['verb_index']) - 1
tokens = dep[self.field].split()[:v + 1]
ints = [self.vocab_to_ints[x] for x in tokens]
try:
ungram = self.vocab_to_ints[self.inflect_verb[tokens[v]]]
except KeyError: # reinflected form not in vocabulary: ignore
continue
n = len(ints) - 1
inp[0, -n:] = ints[:-1]
gram_loss = self.model.test_on_batch(inp, np.array([ints[v]]))
ungram_loss = self.model.test_on_batch(inp, np.array([ungram]))
recs.append((gram_loss, ungram_loss, gram_loss < ungram_loss) +
tuple(dep[x] for x in dependency_fields))
if i % 16 == 0:
progbar.update(i)
self.test_results = pd.DataFrame(recs, columns=columns)
def train(self, n_epochs=10):
if not hasattr(self, 'model'):
self.create_model()
self.compile_model()
self.serialize_class_data()
self.serialize_model()
validation_split = 0.1
split_at = int(len(self.X_train) * (1. - validation_split))
x, val_x = self.X_train[:split_at], self.X_train[split_at:]
y, val_y = self.Y_train[:split_at], self.Y_train[split_at:]
training_loss_history = []
validation_loss_history = []
for epoch in range(n_epochs):
print 'Epoch', epoch
training_loss = []
end = int(float(len(x)) / self.batch_size)
progbar = Progbar(end)
for i in range(0, len(x), self.batch_size):
inp = sequence.pad_sequences(x[i:i + self.batch_size],
maxlen=self.maxlen)
out = y[i:i + self.batch_size]
loss = self.model.train_on_batch(inp, out)
training_loss.append(loss)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
# test on validation set
validation_loss = []
print
print 'Evaluating on validation set:'
end = int(float(len(val_x)) / self.batch_size)
progbar = Progbar(end)
for i in range(0, len(val_x), self.batch_size):
inp = sequence.pad_sequences(val_x[i:i + self.batch_size],
maxlen=self.maxlen)
out = val_y[i:i + self.batch_size]
output = self.model.test_on_batch(inp, out)
validation_loss.append(output)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
training_loss_history.append(np.mean(training_loss))
validation_loss_history.append(np.mean(validation_loss))
filename = op.join(self.serialization_dir,
'weights_epoch%d.h5' % epoch)
self.model.save_weights(filename, overwrite=True)
print
print('Mean training loss: %5.3f; mean validation loss: %5.3f\n' %
(training_loss_history[-1], validation_loss_history[-1]))
if (len(validation_loss_history) > 1
and validation_loss_history[-1] >=
validation_loss_history[-2]):
break
self.training_history = (map(float, training_loss_history),
map(float, validation_loss_history))
def evaluate(self, howmany=1000):
self.model.model._make_test_function()
random.seed(0)
shuffled = self.deps_test[:]
random.shuffle(shuffled)
shuffled = shuffled[:howmany]
X_test = []
Y_test = []
for dep in shuffled:
tokens = self.process_single_dependency(dep)
ints = []
for token in tokens:
if token not in self.vocab_to_ints:
# zero is for pad
x = self.vocab_to_ints[token] = len(self.vocab_to_ints) + 1
self.ints_to_vocab[x] = token
ints.append(self.vocab_to_ints[token])
first = 1
for i in range(first, len(ints) - 1):
X_test.append(ints[:i])
Y_test.append(ints[i])
test_loss = []
end = int(float(len(X_test) / self.batch_size))
progbar = Progbar(end)
for i in range(0, len(X_test), self.batch_size):
inp = sequence.pad_sequences(X_test[i:i + self.batch_size],
maxlen=self.maxlen)
out = Y_test[i:i + self.batch_size]
output = self.model.test_on_batch(inp, out)
test_loss.append(output)
j = int(float(i) / self.batch_size)
if j % 16 == 0:
progbar.update(j)
progbar.update(end)
return np.mean(test_loss)
X_train = np.reshape(X_train, (X_train.shape[0], 1, X_train.shape[1]))
X_test = np.reshape(X_test, (X_test.shape[0], 1, X_test.shape[1]))
model = Sequential()
model.add(SimpleRNN(units=128, input_shape=(1, step), activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(64, activation="relu"))
model.add(Dense(32, activation="relu"))
model.add(Dropout(0.05))
model.add(Dense(32, activation="relu"))
model.add(Dense(16, activation="relu"))
model.add(Dense(8, activation="relu"))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
model.summary()
#모델 학습
optimizer_history = model.fit(X_train,
y_train,
epochs=300,
batch_size=1500,
verbose=1,
validation_data=(X_test, y_test))
#모델 평가
results = model.evaluate(X_test, y_test)
print('loss: ', results[0])
model.add(Dropout(0.25))
# Слой преобразования данных из 2D представления в плоское
model.add(Flatten())
# Полносвязный слой для классификации
model.add(Dense(512, activation='relu'))
# Слой регуляризации Dropout
model.add(Dropout(0.5))
# Выходной полносвязный слой
model.add(Dense(nb_classes, activation='softmax'))
# Задаем параметры оптимизации
# lr = 0.01
lr = 0.02
sgd = SGD(lr=lr, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
# Обучаем модель
model.fit(X_train,
Y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_split=0.1,
shuffle=True)
# Оцениваем качество обучения модели на тестовых данных
scores = model.evaluate(X_test, Y_test, verbose=0)
print("Точность работы на тестовых данных:")
print(round(scores[1] * 100, 3))
name='dense8'))
model.add(BatchNormalization(epsilon=epsilon, momentum=momentum, name='bn8'))
model.add(Activation(binary_tanh, name='act8'))
# dense3
model.add(
BinaryDense(classes,
H=H,
kernel_lr_multiplier=kernel_lr_multiplier,
use_bias=use_bias,
name='dense9'))
model.add(BatchNormalization(epsilon=epsilon, momentum=momentum, name='bn9'))
# In[11]:
opt = Adam(lr=lr_start)
model.compile(loss='squared_hinge', optimizer=opt, metrics=['acc'])
model.summary()
# In[14]:
epochs = 50
lr_scheduler = LearningRateScheduler(lambda e: lr_start * lr_decay**e)
history = model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_test, Y_test),
callbacks=[lr_scheduler])
score = model.evaluate(X_test, Y_test, verbose=0)
print('Test score:', score[0])
def main():
# used to get the session/graph data from keras
K.set_learning_phase(0)
# get the data in a Pandas dataframe
raw_data = pd.read_csv(FLAGS.csv_file)
# convert to one hot vectors
emotion_array = process_emotion(raw_data[['emotion']])
# convert to a 48x48 float matrix
pixel_array = process_pixels(raw_data[['pixels']])
# split for test/train
y_train, y_test = split_for_test(emotion_array)
x_train_matrix, x_test_matrix = split_for_test(pixel_array)
n_train = int(len(x_train_matrix))
n_test = int(len(x_test_matrix))
x_train_input = duplicate_input_layer(x_train_matrix, n_train)
x_test_input = duplicate_input_layer(x_test_matrix, n_test)
# vgg 16. include_top=False so the output is the 512 and use the learned weights
vgg16 = VGG16(include_top=False, input_shape=(48, 48, 3), pooling='avg', weights='imagenet')
# get vgg16 outputs
x_train_feature_map = get_vgg16_output(vgg16, x_train_matrix, n_train)
x_test_feature_map = get_vgg16_output(vgg16, x_test_matrix, n_test)
# build and train model
top_layer_model = Sequential()
top_layer_model.add(Dense(256, input_shape=(512,), activation='relu'))
top_layer_model.add(Dense(256, input_shape=(256,), activation='relu'))
top_layer_model.add(Dropout(0.5))
top_layer_model.add(Dense(128, input_shape=(256,)))
top_layer_model.add(Dense(NUM_CLASSES, activation='softmax'))
adamax = Adamax()
top_layer_model.compile(loss='categorical_crossentropy',
optimizer=adamax, metrics=['accuracy'])
# train
top_layer_model.fit(x_train_feature_map, y_train,
validation_data=(x_train_feature_map, y_train),
nb_epoch=FLAGS.n_epochs, batch_size=FLAGS.batch_size)
# Evaluate
score = top_layer_model.evaluate(x_test_feature_map,
y_test, batch_size=FLAGS.batch_size)
print("After top_layer_model training (test set): {}".format(score))
# Merge two models and create the final_model_final_final
inputs = Input(shape=(48, 48, 3))
vg_output = vgg16(inputs)
print("vg_output: {}".format(vg_output.shape))
# TODO: the 'pooling' argument of the VGG16 model is important for this to work otherwise you will have to squash
# output from (?, 1, 1, 512) to (?, 512)
model_predictions = top_layer_model(vg_output)
final_model = Model(input=inputs, output=model_predictions)
final_model.compile(loss='categorical_crossentropy',
optimizer=adamax, metrics=['accuracy'])
final_model_score = final_model.evaluate(x_train_input,
y_train, batch_size=FLAGS.batch_size)
print("Sanity check - final_model (train score): {}".format(final_model_score))
final_model_score = final_model.evaluate(x_test_input,
y_test, batch_size=FLAGS.batch_size)
print("Sanity check - final_model (test score): {}".format(final_model_score))
# config = final_model.get_config()
# weights = final_model.get_weights()
# probably don't need to create a new model
# model_to_save = Model.from_config(config)
# model_to_save.set_weights(weights)
model_to_save = final_model
print("Model input name: {}".format(model_to_save.input))
print("Model output name: {}".format(model_to_save.output))
# Save Model
builder = saved_model_builder.SavedModelBuilder(FLAGS.export_path)
signature = predict_signature_def(inputs={'images': model_to_save.input},
outputs={'scores': model_to_save.output})
with K.get_session() as sess:
builder.add_meta_graph_and_variables(sess=sess,
tags=[tag_constants.SERVING],
signature_def_map={'predict': signature})
builder.save()
def myTradingSystem(DATE, OPEN, HIGH, LOW, CLOSE, VOL, exposure, equity, settings):
''' This system uses trend following techniques to allocate capital into the desired equities'''
try:
print(settings['counter'])
except:
settings['counter'] = 0
settings['LSTM_regressor'] = dict()
settings['sc'] = dict()
nMarkets = CLOSE.shape[1]
pos = numpy.zeros(nMarkets)
print(DATE[0])
count_in_cash = 0
for market in range(1,nMarkets):
print("Processing market, index: {}".format(market))
close = CLOSE[:, market]
DATE_ = DATE
data = pd.DataFrame()
print(len(close) == len(DATE_))
data['CLOSE'] = close
data['observation_date'] = DATE_
data['observation_date'] = pd.to_datetime(data['observation_date'].astype(str))
data = data.merge(features, on=date, how="left")
print(data.head())
# retrain the lSTM model every 100 days
if settings['counter']%100==0:
training_set = numpy.reshape(CLOSE[:,market], (CLOSE[:,market].shape[0], 1))
print("training_set", training_set.shape)
training_set = (training_set-numpy.insert(training_set, 0, 0, axis=0)[:-1,])/numpy.insert(training_set, 0, 0, axis=0)[:-1,]
training_set = training_set[1:,]
print(training_set.shape)
sc = MinMaxScaler(feature_range = (0, 1))
training_set_scaled = sc.fit_transform(training_set)
settings['sc'][str(market)] = sc
X_train = []
y_train = []
for i in range(30, training_set.shape[0]):
X_train.append(training_set_scaled[i-30:i,0])
y_train.append(training_set_scaled[i,0])
X_train, y_train = numpy.array(X_train), numpy.array(y_train)
print("len of X_train", len(X_train))
X_train = numpy.reshape(X_train, (X_train.shape[0], X_train.shape[1], 1))
print(X_train.shape)
# LSTM
print("Re-training LSTM!")
regressor = Sequential()
regressor.add(LSTM(units = 25, return_sequences = True, input_shape = (X_train.shape[1], 1)))
regressor.add(Dropout(0.1))
regressor.add(LSTM(units = 20, return_sequences = True))
regressor.add(Dropout(0.1))
regressor.add(LSTM(units = 5))
regressor.add(Dropout(0.1))
regressor.add(Dense(units = 1))
regressor.compile(optimizer = 'Adam', loss = 'mean_squared_error')
regressor.fit(X_train, y_train, epochs = 1, batch_size = 32)
settings['LSTM_regressor'][str(market)] = regressor
print("Completed re-training!")
else:
print("Deploying existing LSTM!")
sc = settings['sc'][str(market)]
regressor = settings['LSTM_regressor'][str(market)]
X_pred = []
pred_set = numpy.reshape(CLOSE[:,market], (CLOSE[:,market].shape[0], 1))
pred_set = (pred_set - numpy.insert(pred_set, 0, 0, axis=0)[:-1,])/numpy.insert(pred_set, 0, 0, axis=0)[:-1,]
pred_set = pred_set[1:,]
pred_set_scaled = sc.fit_transform(pred_set)
X_pred.append(pred_set_scaled[-30:,0])
X_pred = numpy.array(X_pred)
X_pred = numpy.reshape(X_pred, (X_pred.shape[0], X_pred.shape[1], 1))
predicted_stock_price = regressor.predict(X_pred)
print(predicted_stock_price)
predicted_stock_price = sc.inverse_transform(predicted_stock_price)
err_term = 0.005 #%[0.005:0.001:0.008]#
if predicted_stock_price[0][0] > 0:
pos[market] = 1
# print('LONG')
elif predicted_stock_price[0][0] + err_term < 0:
pos[market] = -1
# print('SHORT')
else:
pos[0] = pos[0] + 1
print('*' * 100)
settings['counter'] = settings['counter'] + 1
return pos, settings
def compile():
single = Sequential()
single.add(
Conv2D(64, (7, 7),
strides=(1, 1),
padding='valid',
activation='relu',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
input_shape=(33, 33, 4)))
single.add(BatchNormalization())
single.add(Dropout(0.5))
single.add(
Conv2D(128, (5, 5),
strides=(1, 1),
padding='valid',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
activation='relu'))
single.add(BatchNormalization())
single.add(Dropout(0.5))
single.add(
Conv2D(128, (5, 5),
strides=(1, 1),
padding='valid',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
activation='relu'))
single.add(BatchNormalization())
single.add(Dropout(0.5))
single.add(
Conv2D(128, (3, 3),
strides=(1, 1),
padding='valid',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
activation='relu'))
single.add(BatchNormalization())
single.add(Dropout(0.25))
single.add(
Conv2D(128, (3, 3),
strides=(1, 1),
padding='valid',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
activation='relu'))
single.add(Dropout(0.25))
single.add(
Conv2D(128, (3, 3),
strides=(1, 1),
padding='valid',
kernel_regularizer=l1_l2(l1=0.01, l2=0.01),
activation='relu'))
single.add(BatchNormalization())
single.add(Dropout(0.25))
single.add(Flatten())
single.add(Dense(5, activation='softmax'))
sgd = SGD(lr=0.001, decay=0.01, momentum=0.9)
single.compile(loss='categorical_crossentropy', optimizer='sgd')
return single
test_accuracy = []
for i in range(0,50):
model = Sequential()
model.add(Conv2D(32, kernel_size=(3, 3),
activation='relu',
input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,#keras.losses.mean_squared_error,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
test_loss.append(score[0])
test_accuracy.append(score[1])
np.save('./test_loss_CNN_permute',test_loss)
np.save('./test_accuracy_CNN_permute',test_accuracy)
def create_model():
model = Sequential()
model.add(LSTM(units=4, input_shape=(1, look_back)))
model.add(Dense(units=1))
model.compile(loss='mean_squared_error', optimizer='adam')
return model
model.add(Conv2D(1256, (3, 3), padding='same', activation='relu'))
model.add(Conv2D(1256, (1, 1), padding='same', activation='relu'))
model.add(MaxPooling2D(pool_size=(3, 3), strides=(2, 2), padding='valid'))
#2x2x1256
model.add(AveragePooling2D(pool_size=(2, 2), strides=(1, 1), padding='valid'))
#1x1x1256
model.add(Flatten())
#model.add(Dense(4096, activation='relu', use_bias=True))
model.add(Dense(628, activation='relu', use_bias=True))
model.add(Dense(314, activation='relu', use_bias=True))
model.add(Dense(5, activation='softmax', use_bias=True))
print("Model created!!")
model.summary()
print("\nCompiling model...")
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
sgd = SGD(momentum=0.9, nesterov=True, lr=0.003)
#callbacks = [LearningRateScheduler(8, verbose=1)]
model.compile(optimizer='sgd',
loss='categorical_crossentropy',
metrics=['accuracy'])
print("Successfully compiled model!!")
model.save("WorkingModels/convnet224x224x3_untrained.h5")
print("Model saved!!")
plt.show()
print("a")
#学習モデルの作成
in_out_neurons = 1
hidden_neurons = 300
pat = 1
early_stopping = EarlyStopping(monitor='val_loss', patience=pat, verbose=1)
csv = CSVLogger('./Training_patience_' + str(pat) + '.csv')
model = Sequential()
model.add(LSTM(hidden_neurons, batch_input_shape=(None, maxlen, in_out_neurons), return_sequences=False))
model.add(Dense(in_out_neurons))
model.add(Activation("linear"))
model.compile(loss="mean_squared_error", optimizer="rmsprop")
fit = model.fit(X_train, Y_train, batch_size=512, epochs=10000,
validation_data=(X_validation, Y_validation), callbacks=[early_stopping, csv])
#予測
n_in = len(X[0][0])
original = x_lin[len(x_lin)-500:len(x_lin)-500+maxlen]
Z = original.reshape(1, 150, -1)
predicted = [None for i in range(maxlen)]
for i in range(length_of_sequences - maxlen + 1):
z_ = Z[-1:]
y_ = model.predict(z_)
sequence_ = np.concatenate((z_.reshape(maxlen, n_in)[1:], y_),axis=0)
sequence_ = sequence_.reshape(1, maxlen, n_in)
Z = np.append(Z, sequence_, axis=0)
parser = argparse.ArgumentParser()
parser.add_argument("--datapath",help="path to the pickled data",default=save_path)
args = parser.parse_args()
data_path = args.datapath
with open(data_path,"r") as f:
(X,y) = pickle.load(f)
(X,y) = shuffle(X,y)
s=720
Xtr,ytr = X[:s],y[:s]
Xt,yt = X[s:],y[s:]
input_shape = X.shape[1:]
model = Sequential()
model.add(Conv2D(16,(7,7),strides=(3,3),activation="relu", input_shape=input_shape))
model.add(Dropout(0.1))
model.add(Conv2D(16,(3,3),strides=(1,1),activation="relu"))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(1,activation="sigmoid"))
model.compile(loss="binary_crossentropy",optimizer=Adam(),metrics=["accuracy"])
roc_curves,safeset_percents = k_fold_crossvalidation(X,y,10,model,epochs=14)
aucs = [area_under_curve(f,p) for (f,p) in roc_curves.values()]
print("Got average safeset % of {:.2f}. min :{:.2f}, max: {:.2f}, std: {:.2f}".format(np.mean(safeset_percents),min(safeset_percents),max(safeset_percents),np.std(safeset_percents)))
print("Got average ROC AUC of {:.2f}. min :{:.2f}, max: {:.2f}, std: {:.2f}".format(np.mean(aucs),min(aucs),max(aucs),np.std(aucs)))
plot_crossval_auc(roc_curves)
model.add(Dropout(0.25))
model.add(Conv2D(64, (3,3), activation='relu', padding='same'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes, activation='softmax'))
model.summary()
#훈련(얼리스타핑+체크포인트)
es = EarlyStopping(monitor='val_loss', patience=13, mode='auto')
modelpath = './miniprojectdata/checkpoint/cp-{epoch:02d}-{val_loss:.4f}.hdf5'
cp = ModelCheckpoint(filepath=modelpath, monitor='val_loss', verbose=1, save_best_only=True, save_weights_only=False)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['acc'])
hist = model.fit(x_train, y_train, epochs=35, batch_size=4, verbose=1, validation_split=0.2,callbacks=[es, cp])
#평가
loss, acc = model.evaluate(x_test, y_test, batch_size=2)
print("loss :", loss)
print("acc :", acc)
## pyplot 시각화
y_vloss = hist.history['val_loss']
y_loss = hist.history['loss']
y_vacc = hist.history['val_acc']
y_acc = hist.history['acc']
x_len1 = np.arange(len(y_loss))
x_len2 = np.arange(len(y_acc))
plt.figure(figsize=(6,6))
## 1 Loss 그래프
plt.subplot(2,1,1)
model.add(Convolution2D(nb_filters*2, nb_conv, nb_conv))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(nb_pool, nb_pool)))
model.add(Dropout(0.25))
c = 10
Weight_Decay = c / float(X_train.shape[0])
model.add(Flatten())
model.add(Dense(128, W_regularizer=l2(Weight_Decay)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam')
hist = model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=1, validation_data=(X_valid, Y_valid))
Train_Result_Optimizer = hist.history
Train_Loss = np.asarray(Train_Result_Optimizer.get('loss'))
Train_Loss = np.array([Train_Loss]).T
Valid_Loss = np.asarray(Train_Result_Optimizer.get('val_loss'))
Valid_Loss = np.asarray([Valid_Loss]).T
Train_Acc = np.asarray(Train_Result_Optimizer.get('acc'))
Train_Acc = np.array([Train_Acc]).T
Valid_Acc = np.asarray(Train_Result_Optimizer.get('val_acc'))
Valid_Acc = np.asarray([Valid_Acc]).T
Pool_Train_Loss = Train_Loss
Pool_Valid_Loss = Valid_Loss
Pool_Train_Acc = Train_Acc
model.add(Bidirectional(LSTM(lstm_num_hidden,return_sequences=True)))
#model.add(Bidirectional(GRU(lstm_num_hidden,return_sequences=True)))
#model.add(Bidirectional(GRU(lstm_num_hidden,return_sequences=True)))
model.add(AttentionLayer(hidden_layers=(75,)))
model.add(Dropout(0.5))
model.add(Dense(100,activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(2,activation='softmax'))
model.summary()
optimizer=SGD(learning_rate,decay=1e-6,momentum=0.9,nesterov=True,clipnorm=1.)
metrics=['accuracy','categorical_crossentropy']
loss='categorical_crossentropy'
checkpoint=ModelCheckpoint('bestmodels/cnn_dac_dbgru_da_te',monitor='val_acc',save_best_only=True,mode='max',verbose=1)
model.compile(optimizer=optimizer,loss=loss,metrics=metrics)
max_f1=0
for i in range(20):
model.fit(trainX,trainY,epochs=1,batch_size=24)
# model.save('epochmodels/cnn_dac_dbgru_da_te_'+str(i))
#val_f1=F1(valY,model.predict(valX))
#print 'Epoch',i,': Validation F1 score:',val_f1
print 'Epoch',i
print 'Validation'
print_result(valY,model.predict(valX))
print 'Test'
print_result(testY,model.predict(testX))
print ''
continue
regressor.add(input_1) regressor.add(LSTM(200, return_sequences = True)) regressor.add(LSTM(200, return_sequences = True)) regressor.add(LSTM(100, return_sequences = False)) # print(regressor.get_layer(index=0)) regressor.add(Dense(1)) learning_rate = optimizers.SGD(lr=0.001) regressor.compile(optimizer = 'adam', loss = 'mae') print() regressor.fit(x_train, y_train, epochs = 200, batch_size = 52, verbose=2, shuffle=True) # print(x_train[-52:]) yhat = regressor.predict([x_train[-52:], y_train[-52:]]) print(yhat) test_set['Generated'] = yhat test_set
#生成X和y矩阵
dataMat = np.array(data)
X = dataMat[:, 0:1] # 变量x
y = dataMat[:, 1] #变量y
print(dataMat.shape)
print(X.shape)
print(y.shape)
# 构建神经网络模型
model = Sequential()
model.add(Dense(input_dim=1, units=1))
# 选定loss函数和优化器
model.compile(loss='mse', optimizer='sgd', context=["gpu(0)"])
#model.compile(loss='mse', optimizer='sgd')
#如果是mxnet 指定 context=["gpu(0)"] 可以使用gpu
# 训练过程
print('Training -----------')
for step in range(1001):
cost = model.train_on_batch(X, y)
if step % 50 == 0:
print("After %d trainings, the cost: %f" % (step, cost))
# 测试过程
print('\nTesting ------------')
cost = model.evaluate(X, y, batch_size=40)
print('test cost:', cost)
W, b = model.layers[0].get_weights()
print('Weights=', W, '\nbiases=', b)
def build_model(self, last_activation, loss, lr, output_dim):
'''
used to initialize actor and critic models
generates different models based on the parameters passed
'''
kernel_initializer = keras.initializers.VarianceScaling(
scale=1.0, mode='fan_in', distribution="normal", seed=None)
bias_initializer = "zeros"
model = Sequential()
model.add(
Conv2D(input_shape=self.state_dim,
data_format="channels_last",
filters=16,
kernel_size=(8, 8),
strides=(4, 4),
padding="same",
activation="relu",
kernel_initializer=kernel_initializer))
model.add(
Conv2D(data_format="channels_last",
filters=32,
kernel_size=(6, 6),
strides=(2, 2),
padding="same",
activation="relu",
kernel_initializer=kernel_initializer))
model.add(
Conv2D(data_format="channels_last",
filters=32,
kernel_size=(4, 4),
strides=(1, 1),
padding="same",
activation="relu",
kernel_initializer=kernel_initializer))
model.add(
Conv2D(data_format="channels_last",
filters=64,
kernel_size=(4, 4),
strides=(2, 2),
padding="same",
activation="relu",
kernel_initializer=kernel_initializer))
model.add(
Conv2D(data_format="channels_last",
filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding="same",
activation="relu",
kernel_initializer=kernel_initializer))
model.add(Flatten(data_format="channels_last"))
model.add(
Dense(units=512,
activation="relu",
kernel_initializer=kernel_initializer,
bias_initializer=bias_initializer))
model.add(Dense(output_dim, activation=last_activation))
model.compile(optimizer=Adam(lr=lr), loss=loss)
return model
class Model(BaseEstimator, RegressorMixin):
"""
Model using Keras
"""
def __init__(self, lr=0.1, epochs=300, tokens=None, n=0):
"""
Init of the model layers
:param lr: learning rate
:type lr: float
:param epochs: number of epochs
:type epochs: int
:param tokens: tokens used to train the RNN
:type tokens: list of str
:param n: input_length
:type n: int
"""
if tokens is None:
tokens = ['\n', '&', 'C', '1', '2', '3', '4', 'O', 'N', '(', ')', '=', 'c', '[nH]', 'S', 'n', 's', 'o', '#', 'Cl', '[NH]']
self.lr = lr
self.epochs = epochs
self.number = datetime.datetime.now()
self.estimator = Sequential()
self.estimator.add(Embedding(input_dim=len(tokens), output_dim=len(tokens), input_length=n, mask_zero=False))
self.estimator.add(GRU(units=256, return_sequences=True, activation="tanh", input_shape=(81, 26)))
self.estimator.add(Dropout(0.2))
self.estimator.add(GRU(256, activation='tanh', return_sequences=True))
self.estimator.add(Dropout(0.2))
self.estimator.add(TimeDistributed(Dense(len(tokens), activation='softmax')))
self.tensorboard = None
self.early_stopping = None
self.mcp_save = None
def fit(self, x, y):
"""
Compile and fit the model
:param x: training data
:type x: list of list of int
:param y: target data
:type y: list of list of int
:return: None
"""
optimizer = Adam(lr=self.lr)
self.estimator.summary()
save_directory = 'rnn_models/' + "test" + '/'
log = save_directory + 'log'
model_weights = save_directory + 'model_weights_' + str(self.number) + '.h5'
# model_architecture = save_directory + 'model_architecture_' + str(number) + '.json'
if not os.path.isdir(save_directory):
os.mkdir(save_directory)
if not os.path.isdir(log):
os.mkdir(log)
self.tensorboard = TensorBoard(
log_dir=log + "/{}".format(str(datetime.datetime.now()) + '_model' + str(self.number)))
self.early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
self.mcp_save = ModelCheckpoint(model_weights, save_best_only=True, monitor='val_loss', mode='min')
self.estimator.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
self.estimator.fit(x, y, epochs=self.epochs, batch_size=512, validation_split=0.1,
callbacks=[self.tensorboard, self.early_stopping, self.mcp_save])
def score(self, x, y, sample_weight=None):
"""
Score the model
:param x: training data
:type x: list of list of int
:param y: target data
:type y: list of list of int
:param sample_weight:
:return: Score for the model
"""
score = self.estimator.evaluate(x, y)
print(score)
return score[1]
# ANN # Libraries import keras from keras.models import Sequential from keras.layers import Dense classifier=Sequential() classifier.add(Dense(activation="relu", input_dim=5, units=3, kernel_initializer="uniform")) classifier.add(Dense(activation="relu", units=3, kernel_initializer="uniform")) classifier.add(Dense(activation="sigmoid", units=1, kernel_initializer="uniform")) classifier.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy']) classifier.fit(x=X,y=Y,batch_size=4,epochs=100) y_pred=classifier.predict(X_test) from pandas import DataFrame for i in range(0,418): y_pred[i]=y_pred[i]>0.5 Z=pd.DataFrame(data=dataset_test.iloc[:,0].values,columns=['PassengerId']) outputframe=pd.DataFrame(data=y_pred,columns=['Survived'])
def fit_model(train_data, valid_data, vocab_size, train_portion, time_step = 30, batch_size = 30):
train_data = train_data[0:int(train_data.shape[0]*train_portion)]
train_x, train_y = train_data[0:-1], train_data[1:]
trunc_step = time_step*batch_size
trun_index = train_x.shape[0]//trunc_step*trunc_step
train_x = train_x[0:trun_index]
train_y = train_y[0:trun_index]
#split subsequence and reshape
train_x = np.reshape(train_x, (1, len(train_x)))
train_y = np.reshape(train_y, (1, len(train_y)))
train_x = np.split(train_x, batch_size, axis = 1)
train_y = np.split(train_y, batch_size, axis = 1)
for i in range(batch_size):
train_x[i] = np.split(train_x[i], train_x[i].shape[1]//time_step, axis = 1)
train_y[i] = np.split(train_y[i], train_y[i].shape[1]//time_step, axis = 1)
train_x = np.concatenate(train_x, axis = 1)
train_y = np.concatenate(train_y, axis = 1)
train_x = train_x.reshape(train_x.shape[0]*train_x.shape[1], train_x.shape[2])
#train_x = train_x.reshape(train_x.shape[0]*train_x.shape[1], train_x.shape[2], 1)
#train_y = train_y.reshape(train_y.shape[0]*train_y.shape[1], train_y.shape[2], 1)
train_y = train_y.reshape(train_y.shape[0]*train_y.shape[1], train_y.shape[2])
train_y = np.array([train_y[i][len(train_y[i])-1] for i in range(train_y.shape[0])])
train_y = train_y.reshape(len(train_y), 1)
#validation data prepare
valid_x, valid_y = valid_data[0:-1], valid_data[1:]
trun_index = valid_x.shape[0]//trunc_step*trunc_step
valid_x = valid_x[0:trun_index]
valid_y = valid_y[0:trun_index]
valid_x = np.reshape(valid_x, (1, len(valid_x)))
valid_y = np.reshape(valid_y, (1, len(valid_y)))
valid_x = np.split(valid_x, batch_size, axis = 1)
valid_y = np.split(valid_y, batch_size, axis = 1)
for i in range(batch_size):
valid_x[i] = np.split(valid_x[i], valid_x[i].shape[1]//time_step, axis = 1)
valid_y[i] = np.split(valid_y[i], valid_y[i].shape[1]//time_step, axis = 1)
valid_x = np.concatenate(valid_x, axis = 1)
valid_y = np.concatenate(valid_y, axis = 1)
valid_x = valid_x.reshape(valid_x.shape[0]*valid_x.shape[1], valid_x.shape[2])
#valid_x = valid_x.reshape(valid_x.shape[0]*valid_x.shape[1], valid_x.shape[2], 1)
#valid_y = valid_y.reshape(valid_y.shape[0]*valid_y.shape[1], valid_y.shape[2], 1)
valid_y = valid_y.reshape(valid_y.shape[0]*valid_y.shape[1], valid_y.shape[2])
valid_y = np.array([valid_y[i][len(valid_y[i])-1] for i in range(valid_y.shape[0])])
valid_y = valid_y.reshape(len(valid_y), 1)
print(train_x.shape)
print(valid_x.shape)
print(train_y.shape)
print(valid_y.shape)
model = Sequential()
model.add(Embedding(vocab_size, 120, batch_input_shape = (batch_size, train_x.shape[1])))
model.add(CuDNNLSTM(256, stateful = True))
#model.add(CuDNNLSTM(256, batch_input_shape = (batch_size, train_x.shape[1], 1), return_sequences = True, stateful = True))
model.add(Dropout(0.2))
model.add(Dense(vocab_size, activation = "softmax"))
csv_logger = CSVLogger("ptb_words.csv", append=True, separator=',')
checkpoint = ModelCheckpoint('ptb_words_best.h5', monitor = 'loss', verbose=1, save_best_only = True, mode='min')
adam_opt = Adam(lr = 0.001)
model.compile(loss='sparse_categorical_crossentropy',optimizer=adam_opt,metrics=['accuracy', perplexity])
model.summary()
for i in range(100):
model.fit(train_x, train_y, validation_data=(valid_x, valid_y), batch_size=batch_size, epochs=1, verbose=1, shuffle=False, callbacks=[csv_logger, checkpoint])
model.reset_states()
# 2. 모델
model = Sequential()
model.add(LSTM(64, input_shape=(4, 2), activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dense(1))
model.summary()
# 3. 컴파일(훈련준비),실행(훈련)
model.compile(optimizer='adam', loss='mse', metrics=['mse'])
hist = model.fit(x_train,
y_train,
epochs=30,
batch_size=2,
callbacks=[],
verbose=2,
validation_split=0.03)
plt.figure(figsize=(10, 6)) # -> 도화지의 크기? 출력되는 창의 크기인가 그래프의 크기인가
plt.subplot(2, 1, 1) # 2행1열의 첫번쨰 그림을 그린다.
plt.title('keras80 loss plot')
plt.plot(hist.history['loss'], marker='.', c='red', label='loss')
plt.plot(hist.history['val_loss'], marker='.', c='blue', label='val_loss')
# Convert class vectors to binary class matrices
y_train = np_utils.to_categorical(y_train, nb_classes)
y_test = np_utils.to_categorical(y_test, nb_classes)
model = Sequential()
model.add(Dense(128, input_dim=784))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
model.compile(optimizer="sgd", loss="categorical_crossentropy", metrics=["acc"])
def test_serialization():
spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
spark_model.save("test.h5")
recov = load_spark_model("test.h5")
def test_spark_model_end_to_end(spark_context):
rdd = to_simple_rdd(spark_context, x_train, y_train)
# sync epoch
spark_model = SparkModel(model, frequency='epoch', mode='synchronous', num_workers=2)
spark_model.fit(rdd, epochs=epochs, batch_size=batch_size, verbose=2, validation_split=0.1)
score = spark_model.master_network.evaluate(x_test, y_test, verbose=2)
def train_rnn(config, data, tokens, number):
"""
Train one RNN, keep the best weights of the model and stop it when it doesnt learn anymore
:param config: config to use
:type config: dict
:param data: data to use to train the RNN
:type data: list of list of str
:param tokens: list of tokens used to train the RNN
:type tokens: list of str
:param number: id of the model
:type number: int
:return: None
"""
print("Model : " + str(number))
x_train, y_train = convert_data_to_numbers(tokens, data)
print("SMILES converted to numbers")
maxlen = 81
x = sequence.pad_sequences(x_train, maxlen=maxlen, dtype='int32',
padding='post', truncating='pre', value=0.)
y = sequence.pad_sequences(y_train, maxlen=maxlen, dtype='int32',
padding='post', truncating='pre', value=0.)
print("Loading y_train_one_hot")
y_train_one_hot = np.array([to_categorical(y_i, num_classes=len(tokens)) for y_i in y])
print(y_train_one_hot.shape)
n = x.shape[1]
model = Sequential()
model.add(Embedding(input_dim=len(tokens), output_dim=len(tokens), input_length=n, mask_zero=False))
model.add(GRU(units=256, return_sequences=True, activation="tanh", input_shape=(81, 26)))
model.add(Dropout(0.2))
model.add(GRU(256, activation='tanh', return_sequences=True))
model.add(Dropout(0.2))
model.add(TimeDistributed(Dense(len(tokens), activation='softmax')))
optimizer = Adam(lr=config['learning_rate'])
model.summary()
save_directory = 'rnn_models/' + config['configuration_name'] + '/'
log = save_directory + 'log'
model_weights = save_directory + 'model_weights_' + str(number) + '.h5'
model_architecture = save_directory + 'model_architecture_' + str(number) + '.json'
if not os.path.isdir(save_directory):
os.mkdir(save_directory)
if not os.path.isdir(log):
os.mkdir(log)
tensorboard = TensorBoard(log_dir=log + "/{}".format(str(datetime.datetime.now()) + '_model' + str(number)))
early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0, mode='min')
mcp_save = ModelCheckpoint(model_weights, save_best_only=True, monitor='val_loss', mode='min')
model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'])
model.fit(x, y_train_one_hot, epochs=config['epochs'], batch_size=512,
validation_split=0.1, callbacks=[tensorboard, early_stopping, mcp_save])
model_json = model.to_json()
with open(model_architecture, "w") as json_file:
json_file.write(model_json)
print("Model saved")
with open(save_directory + "config.json", 'w') as conf:
json.dump(config, conf)
print('x train shape', x_train.shape)
print('y train shape', y_train.shape)
#Small dataset for illustration
x_train_small = x_train[:1000]
y_train_small = y_train[:1000]
# DEFINE THE NEURAL NETWORK WITH KERAS
# hidden layers = 3 and output
model_1 = Sequential([
Dense(units=128, input_dim=TOTAL_Inputs, activation='relu', name='hidden_1'),
Dense(units=64, activation='relu', name='hidden_2'), #no need to input dimensions, keras can get it anyway
Dense(16, activation='relu', name='hidden_3'), #16 = units(neurons), its more often in docs. so good practice
Dense(10,activation='softmax', name='output') #output
])
model_1.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
display(model_1.summary())
model_2 = Sequential() # adding dropout - its randomly taking out a neuron
model_2.add(Dropout(0.2, seed=42, input_shape=(TOTAL_Inputs,)))
model_2.add(Dense(128, activation='relu', name='Model_2_hidden_1'))
model_2.add(Dense(64, activation='relu', name='Model_2_hidden_2'))
model_2.add(Dense(16, activation='relu', name='Model_2_hidden_3'))
model_2.add(Dense(10, activation='softmax', name='Model_2_output'))
model_2.compile(optimizer='adam',loss='sparse_categorical_crossentropy',metrics=['accuracy'])
model_3 = Sequential()
model_3.add(Dropout(0.2, seed=42, input_shape=(TOTAL_Inputs,)))
model_3.add(Dense(128, activation='relu', name='Model_3_hidden_1'))
model_3.add(Dropout(0.25, seed=42))
model_3.add(Dense(64, activation='relu', name='Model_3_hidden_2'))
def get_model(data_type):
# Model parameters and other things based upon the type of data
op_dims = 4
op_activ = 'softmax'
# krnl_init = keras.initializers.RandomNormal(mean = 0.0, stddev = 0.05, seed = 11211)
krnl_init = keras.initializers.GlorotUniform(seed = 11211)
bias_init = keras.initializers.Zeros()
if (data_type == 'encoded'):
# NOT IN USE!!!!
ip_dims = 6
l1_units = 200
l2_units = 200
activ = 'relu'
lrn_rt = 1.0e-06
#https://stackoverflow.com/questions/59737875/keras-change-learning-rate
opt = keras.optimizers.Adam(lr = lrn_rt)
elif (data_type == 'sparse'):
ip_dims = INPUT_SIZE
l1_units = 400
l2_units = 90
activ = 'relu'
lrn_rt = 1.0e-04
# Adagrad
#opt = keras.optimizers.Adagrad(
# learning_rate=lrn_rt, initial_accumulator_value=0.1, epsilon=1e-07,
# name='Adagrad')
# Adadelta optimizer
#opt = keras.optimizers.Adadelta(
# lr = lrn_rt, rho = 0.95, epsilon = 1e-07)
# RMSprop optimizer
#https://stackoverflow.com/questions/59737875/keras-change-learning-rate
# opt = keras.optimizers.RMSprop(
# learning_rate = lrn_rt, rho = 0.9, momentum = 0.0, epsilon = 1e-07,
# centered = False, name = 'RMSprop')
# Adam w/wo amsgrad
opt = keras.optimizers.Adam(lr = lrn_rt, amsgrad=True)
# Nadam
# opt = keras.optimizers.Nadam(
# lr=lrn_rt, beta_1=0.9, beta_2=0.999, epsilon=1e-07, name = 'nadam')
#opt = keras.optimizers.SGD(
# learning_rate=lrn_rt, momentum=0.001, nesterov=True, name='SGD'
# )
# Loss
# loss = keras.losses.KLDivergence()
loss = keras.losses.MeanSquaredError(reduction="auto", name="mean_squared_error")
else:
import sys
sys.exit(0)
# creates a generic neural network architecture
model = Sequential()
model.add(Dense(units = l1_units,
input_dim = ip_dims,
activation = activ,
kernel_initializer = krnl_init,
bias_initializer = bias_init))
model.add(Dense(units = l1_units,
input_dim = ip_dims,
activation = activ,
kernel_initializer = krnl_init,
bias_initializer = bias_init))
model.add(Dense(units = l2_units,
activation= activ,
kernel_initializer = krnl_init,
bias_initializer = bias_init))
model.add(Dense(units = l2_units,
activation= activ,
kernel_initializer = krnl_init,
bias_initializer = bias_init))
# output layer
model.add(Dense(units = op_dims,
activation = op_activ,
kernel_initializer = krnl_init,
bias_initializer = bias_init))
# compile the model using traditional Machine Learning losses and optimizers
model.compile(loss=loss, optimizer=opt, metrics=['accuracy'])
return model
parser = argparse.ArgumentParser()
parser.add_argument("--model",
help="filename for output file",
default="six_bin.alt.leaky.sigmoid",
required=False)
args = parser.parse_args()
num_input_dimensions = 9600
num_neurons_in_layer1 = 1500
num_neurons_in_layer2 = 500
num_output_dimensions = 12
model = Sequential()
model.add(
Dense(num_neurons_in_layer1,
input_dim=num_input_dimensions,
activation='relu'))
model.add(LeakyReLU(alpha=.01))
model.add(Dense(num_neurons_in_layer2, activation='relu'))
model.add(LeakyReLU(alpha=.01))
model.add(Dense(num_output_dimensions, activation='sigmoid'))
# 3) Compile Model
metrics_to_output = ['accuracy']
model.compile(loss='mean_absolute_error',
optimizer='adam',
metrics=metrics_to_output)
model.save(args.model + ".h5")
x_validation=x_validation.reshape(x_validation.shape[0],*im_shape)
#creating the model
cnn_model=Sequential([
Conv2D(filters=32,kernel_size=3, activation="relu",input_shape=im_shape),
MaxPooling2D(pool_size=(2,2)),
Dropout(0.2),
Flatten(),
Dense(32,activation="relu"),
Dense(10,activation="softmax")
])
#compiling
cnn_model.compile(
loss="sparse_categorical_crossentropy",
optimizer=adam(lr=0.001),
metrics=["accuracy"]
)
# training the model
cnn_model.fit(
x_train,y_train,batch_size=batch_size,epochs=3 ,
validation_data=(x_validation,y_validation)
)
score=cnn_model.evaluate(x_test,y_test)
print("test loss: {}".format(score[0]))
print("test accuracy: {}".format(score[1]))
def build_LSTM_model(trainData, trainBatches, testData, testBatches,
windowSize, class_count, numCalls, batch_size):
# Specify number of units
# https://stackoverflow.com/questions/37901047/what-is-num-units-in-tensorflow-basiclstmcell#39440218
num_units = 128
embedding_size = 256
# https://keras.io/callbacks/#earlystopping
early_stop = cb.EarlyStopping(monitor='sparse_categorical_accuracy',
min_delta=0.0001,
patience=3)
model = Sequential()
# We need to add an embedding layer because LSTM (at this moment) that the API call indices (numbers)
# are of some mathematical significance. E.g., system call 2 is "closer" to system calls 3 and 4.
# But system call numbers have nothing to do with their semantic meaning and relation to other
# system calls. So we transform it using an embedding layer so the LSTM can figure these relationships
# out for itself.
# https://blog.keras.io/a-ten-minute-introduction-to-sequence-to-sequence-learning-in-keras.html
# https://stackoverflow.com/questions/40695452/stateful-lstm-with-embedding-layer-shapes-dont-match
api_count = numCalls + 1 # +1 because 0 is our padding number
model.add(
Embedding(input_dim=api_count, output_dim=256,
input_length=windowSize))
# https://keras.io/layers/recurrent/#lstm
# model.add(LSTM(num_units,input_shape=(windowSize, api_count),return_sequences=False))
#TODO - GPU stuffs
model.add(
CuDNNLSTM(num_units,
input_shape=(windowSize, api_count),
return_sequences=False))
# NOTE: If I want to add more layers
# https://stackoverflow.com/questions/40331510/how-to-stack-multiple-lstm-in-keras
# https://keras.io/layers/core/#dense
model.add(Dense(128))
# https://keras.io/activations/
model.add(Activation('relu'))
# https://keras.io/layers/core/#dropout
model.add(Dropout(0.5))
model.add(Dense(class_count, name='logits'))
model.add(Activation('softmax'))
# Which optimizer to use
# https://keras.io/optimizers/
opt = optimizers.RMSprop(lr=0.01, decay=0.001)
# https://keras.io/models/model/#compile
model.compile(
loss='sparse_categorical_crossentropy',
optimizer=opt,
# Metrics to print
# We use sparse_categorical_accuracy as opposed to categorical_accuracy
# because: https://stackoverflow.com/questions/44477489/keras-difference-between-categorical-accuracy-and-sparse-categorical-accuracy
# I.e., since we don't use hot-encoding, we use sparse_categorical_accuracy
metrics=['sparse_categorical_accuracy'])
# https://keras.io/models/model/#fit_generator
hist = model.fit_generator(
# Data to train
trainData,
# Use multiprocessing because python Threading isn't really
# threading: https://docs.python.org/3/glossary.html#term-global-interpreter-lock
use_multiprocessing=True,
# Number of steps per epoch (this is how we train our large
# number of samples dataset without running out of memory)
steps_per_epoch=trainBatches,
#TODO
# Number of epochs
epochs=100,
# Validation data (will not be trained on)
validation_data=testData,
validation_steps=testBatches,
# Do not shuffle batches.
shuffle=False,
# List of callbacks to be called while training.
callbacks=[early_stop])
return model, hist
model.add(Lambda(lambda x: x / 127.5 - 1.0, input_shape=INPUT_SHAPE))
model.add(Conv2D(24, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(36, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(48, 5, 5, activation='elu', subsample=(2, 2)))
model.add(Conv2D(64, 3, 3, activation='elu'))
model.add(Conv2D(64, 3, 3, activation='elu'))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(100, activation='elu'))
model.add(Dense(50, activation='elu'))
model.add(Dense(10, activation='elu'))
model.add(Dense(1))
model.summary()
checkpoint = ModelCheckpoint('model-{epoch:03d}.h5',
monitor='val_loss',
verbose=0,
save_best_only=True,
mode='auto')
model.compile(loss='mean_squared_error', optimizer=Adam(lr=1.0e-4))
model.fit_generator(batch_generator(X_train),
validation_data=batch_generator(X_validation),
nb_val_samples=len(X_validation),
samples_per_epoch=len(X_train),
nb_epoch=10,
verbose=1)
model.save('./model1.h5')