from keras import Input
# build network model with multi-input
text_vocabulary_size = 10000 # the first 10000 words used most frequently; 10000 as the input dim in Embedding layer.
question_vocabulary_size = 10000 # question text input. input one
answer_vocabulary_size = 500 # answer text input. input two
# input 1:
text_input = Input(shape=(None, ), dtype='int32', name='text')
# shape(None,): None dimension specified, The length of array input is uncertain.
# name: the unique layer name in the model.
embedded_text = layers.Embedding(text_vocabulary_size, 64)(
text_input) # out:a word embedded into a 64 dimension vector
encoded_text = layers.LSTM(32)(embedded_text)
# input 2:
question_input = Input(shape=(None, ), dtype='int32',
name='question') # the name should be unique!
embedded_question = layers.Embedding(question_vocabulary_size,
32)(question_input)
encoded_question = layers.LSTM(16)(embedded_question)
# connect two input tensors which could be different size. input 1 is 32 but input 2 is 16.
concatenated = layers.concatenate([encoded_text, encoded_question],
axis=-1) # -1: the last axis
# concatenat: https://blog.csdn.net/leviopku/article/details/82380710
answer = layers.Dense(answer_vocabulary_size, activation='softmax')(
concatenated) # multi-classification
plt.legend()
plt.show()
############### recurrent baseline -GRU (end) ##############
############### another recurrent baseline - LSTM ##############
from keras.models import Sequential
from keras import layers
from keras.optimizers import RMSprop
model = Sequential()
model.add(layers.LSTM(4, dropout=0.5, recurrent_dropout=0.5, input_shape=(None, log_minmax_flux.shape[-1])))
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer=RMSprop(lr = 0.0001), loss='mae')
history = model.fit_generator(train_gen,
steps_per_epoch=2000,
epochs=10,
validation_data=val_gen,
validation_steps=val_steps)
import matplotlib.pyplot as plt
loss = history.history['loss']
val_loss = history.history['val_loss']
encoder_inputs = layers.Input(shape=(None, )) # (1425, 79)
# 임베딩 레이어
encoder_outputs = layers.Embedding(
len(words), embedding_dim, input_length=max_sequences
)(
encoder_inputs
) # (1425, 79) -> (6406, 100, 78) // 마지막 max_sequences는 자동으로 잡아준다? 생략 가능하다. // 80 넣었을 때 에러가 날까 안날까 확인해보기
print(f'encoder_outputs : {encoder_outputs}')
# return_state가 True면 상태값 리턴
# LSTM은 state_h(hidden state)와 state_c(cell state) 2개의 상태 존재
encoder_outputs, state_h, state_c = layers.LSTM(
lstm_hidden_dim, # 128 노드의 개수
dropout=0.1,
recurrent_dropout=0.5,
return_state=True
)(
encoder_outputs
) # (128, 100, 78) (None, 78, 128)
# 히든 상태와 셀 상태를 하나로 묶음
encoder_states = [state_h, state_c]
#--------------------------------------------
# 훈련 모델 디코더 정의
#--------------------------------------------
# 목표 문장의 인덱스 시퀀스를 입력으로 받음
decoder_inputs = layers.Input(shape=(None, )) # x_decoder : (1425, 78)
# 임베딩 레이어
Xtext_val, Ximage_val, ytext_val = preprocessing(dt_val, di_val)
# convert input and output into array
"""RNN model"""
from keras import layers
print(vocab_size)
## image feature
dim_embedding = 64
input_image = layers.Input(shape=(Ximage_train.shape[1], ))
fimage = layers.Dense(256, activation='relu', name="ImageFeature")(input_image)
input_txt = layers.Input(shape=(maxlen, ))
ftxt = layers.Embedding(vocab_size, dim_embedding, mask_zero=True)(input_txt)
ftxt = layers.LSTM(256, name="CaptionFeature")(ftxt)
decoder = layers.add([ftxt, fimage])
decoder = layers.Dense(256, activation='relu')(decoder)
output = layers.Dense(vocab_size, activation='softmax')(decoder)
model = models.Model(inputs=[input_image, input_txt], outputs=output)
model.compile(loss='categorical_crossentropy', optimizer='adam')
print(model.summary())
start = time.time()
hist = model.fit([Ximage_train, Xtext_train],
ytext_train,
epochs=5,
verbose=0,
batch_size=64,
from keras.datasets import imdb
from keras import models
from keras import layers
from Utils import plotHistory, INFO
from keras.preprocessing.sequence import pad_sequences
max_feature = 10000
max_len = 500
print(INFO() + 'loading data')
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_feature)
model = models.Sequential()
model.add(layers.Embedding(max_feature, 32))
model.add(layers.LSTM(32))
model.add(layers.Dense(1, activation='sigmoid'))
model.summary()
print(INFO() + 'training')
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc'])
x_train = pad_sequences(x_train,maxlen=max_len)
history = model.fit(x_train,
y_train,
batch_size=128,
epochs=10,
validation_split=0.2)
plotHistory(history)
print(INFO() + 'end')
print('Validation Data:')
print(x_val.shape)
print(y_val.shape)
# Try replacing GRU, or SimpleRNN.
HIDDEN_SIZE = 128
BATCH_SIZE = 128
LAYERS = 1
EPOCHS = 200
print('Build model...')
model = Sequential()
# "Encode" the input sequence using an RNN, producing an output of HIDDEN_SIZE.
# Note: In a situation where your input sequences have a variable length,
# use input_shape=(None, num_feature).
model.add(layers.LSTM(HIDDEN_SIZE, input_shape=(MAXLEN, len(chars))))
# As the decoder RNN's input, repeatedly provide with the last hidden state of
# RNN for each time step. Repeat 'DIGITS + 1' times as that's the maximum
# length of output, e.g., when DIGITS=3, max output is 999+999=1998.
model.add(layers.RepeatVector(MAXLEN))
# The decoder RNN could be multiple layers stacked or a single layer.
for _ in range(LAYERS):
# By setting return_sequences to True, return not only the last output but
# all the outputs so far in the form of (num_samples, timesteps,
# output_dim). This is necessary as TimeDistributed in the below expects
# the first dimension to be the timesteps.
model.add(layers.LSTM(HIDDEN_SIZE, return_sequences=True))
# Apply a dense layer to the every temporal slice of an input. For each of step
# of the output sequence, decide which character should be chosen.
model.add(layers.TimeDistributed(layers.Dense(len(chars))))
def cnn_2x_lstm_siamese(voc_size, max_len, dropout=0.2):
"""Two siamese branches, each embedding a statement.
Binary classifier on top.
Args:
voc_size: size of the vocabulary for the input statements.
max_len: maximum length for the input statements.
dropout: Fraction of units to drop.
Returns:
A Keras model instance.
"""
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
x = layers.Conv1D(32,
7,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(32,
7,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(64,
5,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(64,
5,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(128,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(128,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
x = layers.Conv1D(256,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.ZeroPadding1D(padding=1)(x)
x = layers.Conv1D(256,
3,
activation='relu',
kernel_regularizer=regularizers.l2(0.01))(x)
x = layers.MaxPooling1D(pool_size=2, strides=2)(x)
embedded_pivot = layers.LSTM(256)(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='relu')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='relu')(x)
model = Model([pivot_input, statement_input], prediction)
return model
# Target features at timestep 2
[105, 106, 107, 108],
# Datapoint 2
# Target features at timestep 2
[205, 206, 207, 208]
])
# Each input data point has 2 timesteps, each with 3 features.
# So the input shape (excluding batch_size) is (2, 3), which
# matches the shape of each data point in data_x above.
model_input = L.Input(shape=(1292,128))
# This RNN will return timesteps with 4 features each.
# Because return_sequences=True, it will output 2 timesteps, each
# with 4 features. So the output shape (excluding batch size) is
# (2, 4), which matches the shape of each data point in data_y above.
model_output = L.LSTM(num_classes, return_sequences=False)(model_input)
# Create the model.
model = M.Model(input=model_input, output=model_output)
# You need to pick appropriate loss/optimizers for your problem.
# I'm just using these to make the example compile.
opt = keras.optimizers.rmsprop(lr=0.0001)
model.compile(optimizer=opt, loss="mean_squared_error", metrics=['accuracy'])
dataPath = '/data/hibbslab/jyang/tzanetakis/ver6.0/'
x_train = pickle.load(open(dataPath + 'x_train_mel.p', 'rb'))
y_train = pickle.load(open(dataPath + 'y_train_mel.p', 'rb'))
x_test = pickle.load(open(dataPath + 'x_test_mel.p', 'rb'))
y_test = pickle.load(open(dataPath + 'y_test_mel.p', 'rb'))
sequence_input = Input(shape=(30,75))
processed_sequence = TimeDistributed(single_model)(sequence_input)
pre_model = Model(inputs=sequence_input, outputs=processed_sequence)
model = models.Sequential()
# model.add(layers.SimpleRNN(
# # unroll=True,
# units=16,
# batch_input_shape=(None, 50, 75),#75相当于特征数量，或者也可以叫维度，这里有3*25个坐标点，要提前整形
# return_sequences=0
# ))
model.add(pre_model)
model.add(layers.LSTM(
units=128,
batch_input_shape=(None, 30, 75), # 75相当于特征数量，或者也可以叫维度，这里有3*25个坐标点，要提前整形
return_sequences=0 # 是否返回整个列表，用于再嵌套LSTM
))
# model.add(layers.Dense(16, activation='relu')) # 这个shape好像是抛去了数据的第一维度，认为这个
# model.add(layers.Dense(16, activation='relu'))
# model.add(layers.Dense(8, activation='relu'))
model.add(layers.Dense(40, activation='relu'))
model.add(layers.Dense(40,activation='softmax'))
model.summary()
# plot_model(model, to_file='model.png')
#编译步骤
model.compile(optimizer='rmsprop',
# loss='binary_crossentropy',
loss='categorical_crossentropy',
metrics=['accuracy'])
def brsmv1(num_features=39,
num_classes=28,
num_hiddens=256,
num_layers=5,
dropout=0.2,
zoneout=0.,
input_dropout=False,
input_std_noise=.0,
weight_decay=1e-4,
residual=None,
layer_norm=None,
mi=None,
activation='tanh'):
""" BRSM v1.0
Improved features:
Reference:
[1] Gal, Y, "A Theoretically Grounded Application of Dropout in
Recurrent Neural Networks", 2015.
[2] Graves, Alex, Abdel-rahman Mohamed, and Geoffrey Hinton. "Speech
recognition with deep recurrent neural networks", 2013.
[3] Krueger, David, et al. "Zoneout: Regularizing rnns by randomly
preserving hidden activations", 2016.
[4] Ba, Jimmy Lei, Jamie Ryan Kiros, and Geoffrey E. Hinton. "Layer
normalization.", 2016.
[5] Wu, Yuhuai, et al. "On multiplicative integration with recurrent
neural networks." Advances In Neural Information Processing Systems.
2016.
[6] Wu, Yonghui, et al. "Google's Neural Machine Translation System:
Bridging the Gap between Human and Machine Translation.", 2016.
"""
x = Input(name='inputs', shape=(None, num_features))
o = x
if input_std_noise is not None:
o = GaussianNoise(input_std_noise)(o)
if residual is not None:
o = TimeDistributed(
Dense(num_hiddens * 2, kernel_regularizer=l2(weight_decay)))(o)
if input_dropout:
o = Dropout(dropout)(o)
for i, _ in enumerate(range(num_layers)):
new_o = Bidirectional(
keras_layers.LSTM(num_hiddens,
return_sequences=True,
kernel_regularizer=l2(weight_decay),
recurrent_regularizer=l2(weight_decay),
dropout=dropout,
recurrent_dropout=dropout,
activation=activation))(o)
if residual is not None:
o = merge([new_o, o], mode=residual)
else:
o = new_o
o = TimeDistributed(Dense(num_classes,
kernel_regularizer=l2(weight_decay)))(o)
return ctc_model(x, o)
lstm_shape = (args.num_img_pair - 1, 3, args.img_height, args.img_width)
lstm_i1 = Input(shape=lstm_shape)
lstm_i2 = Input(shape=lstm_shape)
if not args.resume:
if args.pre_trained:
cnn_model = model.flownetS_to_load(args.img_height, args.img_width)
l = cnn_model.layers
imgs = layers.concatenate([lstm_i1, lstm_i2], axis=2)
x = layers.TimeDistributed(l[1])(imgs)
for i in range(2, 27):
x = layers.TimeDistributed(l[i])(x)
x = layers.TimeDistributed(layers.Flatten())(x)
if not args.bidirectional:
x = layers.LSTM(1000,
return_sequences=True,
dropout=args.dropout,
recurrent_dropout=args.recurrent_dropout)(x)
x = layers.LSTM(1000,
return_sequences=True,
dropout=args.dropout,
recurrent_dropout=args.recurrent_dropout)(x)
else:
x = layers.Bidirectional(
layers.LSTM(1000,
return_sequences=True,
dropout=args.dropout,
recurrent_dropout=args.recurrent_dropout))(x)
x = layers.Bidirectional(
layers.LSTM(1000,
return_sequences=True,
dropout=args.dropout,
dataY = numpy.array(dataY)
print('dataY shape: ', dataY.shape)
#exit()
return dataX, dataY
time_step = 3
trainX, trainY = create_dataset(X_train, time_step)
testX, testY = create_dataset(X_test, time_step)
#Creating the RNN and LSTM Neural Networks
RNN = Sequential()
RNN.add(layers.SimpleRNN(100, input_shape=(time_step, features)))
RNN.add(layers.Dense(1))
LSTM = Sequential()
LSTM.add(layers.LSTM(512, input_shape=(time_step, features)))
LSTM.add(layers.Dense(1))
#Using the mean squared error to determine our training value loss
RNN.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])
LSTM.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])
#establishing the training history for both the RNN and LSTM
history_RNN = RNN.fit(trainX, trainY, validation_data=(testX, testY), epochs=5, batch_size=1)
history_LSTM = LSTM.fit(trainX, trainY, validation_data=(testX, testY), epochs=5, batch_size=1)
#Plotting our training history for both the LSTM and RNN
plt.plot(history_RNN.history['val_loss'], label='val_loss')
plt.plot(history_RNN.history['loss'], label='loss')
plt.title('RNN Model Loss')
print (x_train)
print(x_test)
#Takes the vocabulary size in order to create the matrix of the embending layer
vocab_size=len(tokenizer.word_index)+1
print(vocab_size)
#The data should come as a vectors of the same size, we put 0 at the end of the shorter data
x_train=pad_sequences(x_train,padding='post',maxlen=max_ln)
x_test=pad_sequences(x_test,padding='post',maxlen=max_ln)
#Create the model imput 3 Layers, one is LSTM so our NN is recursive
model1 = Sequential()
embedding_layer = layers.Embedding(vocab_size, max_words, input_length=max_ln, trainable=False)
model1.add(embedding_layer)
model1.add(layers.LSTM(128))
model1.add(layers.Dense(1, activation='sigmoid'))
model1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
#Print Model's summary
print(model1.summary())
#Fiting and training the model in a 100 epochs, the NN inputs data in batches of 128(Input Layer)
history = model1.fit(x_train, y_train, batch_size=128, epochs=100, verbose=1, validation_split=0.2)
#Evaluate Model
score = model1.evaluate(x_test, y_test, verbose=1)
#Print Accuracy and loss
print("Test Score:", score[0])
print("Test Accuracy:", score[1])
height = len(cm[0])
for x in range(width):
for y in range(height):
ax.annotate(str(cm[x][y]), xy=(y, x), horizontalalignment='center',
verticalalignment='center', color=getFontColor(cm[x][y]))
# add genres as ticks
alphabet = mods
plt.xticks(range(width), alphabet[:width], rotation=30)
plt.yticks(range(height), alphabet[:height])
return plt
from keras import layers
from keras import Sequential,layers
model=Sequential()
model.add(layers.LSTM(128,dropout=0.7,return_sequences=True,input_shape=(128,2)))
model.add(layers.LSTM(128,dropout=0.7))
model.add(layers.Dense(len(mods),activation="sigmoid"))
model.compile(loss='categorical_crossentropy', optimizer='adam',metrics=['accuracy'])
nb_epoch = 80 # number of epochs to train on
batch_size = 256 # training batch size
###################################################train the network#################################################
history = model.fit(X_train,
Y_train,
batch_size=batch_size,
epochs=nb_epoch,
verbose=1,
validation_data=(X_test, Y_test)
)
# we re-load the best weights once training is finished
def retain(ARGS):
"""Create the model"""
#Define the constant for model saving
reshape_size = ARGS.emb_size + ARGS.numeric_size
if ARGS.allow_negative:
embeddings_constraint = FreezePadding()
beta_activation = 'tanh'
output_constraint = None
else:
embeddings_constraint = FreezePadding_Non_Negative()
beta_activation = 'sigmoid'
output_constraint = non_neg()
#Get available gpus , returns empty list if none
glist = get_available_gpus()
def reshape(data):
"""Reshape the context vectors to 3D vector"""
return K.reshape(x=data, shape=(K.shape(data)[0], 1, reshape_size))
#Code Input
codes = L.Input((None, None), name='codes_input')
inputs_list = [codes]
#Calculate embedding for each code and sum them to a visit level
codes_embs_total = L.Embedding(
ARGS.num_codes + 1,
ARGS.emb_size,
name='embedding',
embeddings_constraint=embeddings_constraint)(codes)
codes_embs = L.Lambda(lambda x: K.sum(x, axis=2))(codes_embs_total)
#Numeric input if needed
if ARGS.numeric_size:
numerics = L.Input((None, ARGS.numeric_size), name='numeric_input')
inputs_list.append(numerics)
full_embs = L.concatenate([codes_embs, numerics], name='catInp')
else:
full_embs = codes_embs
#Apply dropout on inputs
full_embs = L.Dropout(ARGS.dropout_input)(full_embs)
#Time input if needed
if ARGS.use_time:
time = L.Input((None, 1), name='time_input')
inputs_list.append(time)
time_embs = L.concatenate([full_embs, time], name='catInp2')
else:
time_embs = full_embs
#Setup Layers
#This implementation uses Bidirectional LSTM instead of reverse order
# (see https://github.com/mp2893/retain/issues/3 for more details)
#If training on GPU and Tensorflow use CuDNNLSTM for much faster training
if glist:
alpha = L.Bidirectional(L.CuDNNLSTM(ARGS.recurrent_size,
return_sequences=True),
name='alpha')
beta = L.Bidirectional(L.CuDNNLSTM(ARGS.recurrent_size,
return_sequences=True),
name='beta')
else:
alpha = L.Bidirectional(L.LSTM(ARGS.recurrent_size,
return_sequences=True,
implementation=2),
name='alpha')
beta = L.Bidirectional(L.LSTM(ARGS.recurrent_size,
return_sequences=True,
implementation=2),
name='beta')
alpha_dense = L.Dense(1, kernel_regularizer=l2(ARGS.l2))
beta_dense = L.Dense(ARGS.emb_size + ARGS.numeric_size,
activation=beta_activation,
kernel_regularizer=l2(ARGS.l2))
#Compute alpha, visit attention
alpha_out = alpha(time_embs)
alpha_out = L.TimeDistributed(alpha_dense,
name='alpha_dense_0')(alpha_out)
alpha_out = L.Softmax(axis=1)(alpha_out)
#Compute beta, codes attention
beta_out = beta(time_embs)
beta_out = L.TimeDistributed(beta_dense, name='beta_dense_0')(beta_out)
#Compute context vector based on attentions and embeddings
c_t = L.Multiply()([alpha_out, beta_out, full_embs])
c_t = L.Lambda(lambda x: K.sum(x, axis=1))(c_t)
#Reshape to 3d vector for consistency between Many to Many and Many to One implementations
contexts = L.Lambda(reshape)(c_t)
#Make a prediction
contexts = L.Dropout(ARGS.dropout_context)(contexts)
output_layer = L.Dense(1,
activation='sigmoid',
name='dOut',
kernel_regularizer=l2(ARGS.l2),
kernel_constraint=output_constraint)
#TimeDistributed is used for consistency
# between Many to Many and Many to One implementations
output = L.TimeDistributed(output_layer,
name='time_distributed_out')(contexts)
#Define the model with appropriate inputs
model = Model(inputs=inputs_list, outputs=[output])
return model
# 计算val_steps，这是计算需要从val_gen中抽取多少次
val_steps=(valnum-lookback)//batch_size
# 查看训练集需要抽取的次数
train_steps=trainnum//batch_size
from keras.callbacks import EarlyStopping
early_stopping=EarlyStopping(monitor="val_loss",patience=30)
from keras.callbacks import ReduceLROnPlateau
lr_reduce=ReduceLROnPlateau(monitor="val_loss,factor=0.1,patience=10")
#创建模型
model1=models.Sequential()
model1.add(layers.Dense(512,activation="relu",input_shape=(None,data.shape[-1])))
model1.add(layers.Dropout(0.5))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5,return_sequences=True)))
model1.add(layers.Bidirectional(layers.LSTM(32,activation="relu",dropout=0.5)))
model1.add(layers.Dense(512,activation="relu"))
model1.add(layers.Dropout(0.5))
model1.add(layers.Dense(8,activation="softmax"))
model1.summary()
model1.compile(optimizer=optimizers.RMSprop(),
loss="categorical_crossentropy",
metrics=["acc"])
global_en_mac_tune = 0
global_de_mac_tune = 0
num_iterations = 10
for i in range(num_iterations):
print('training iteration:', i + 1)
# build model
model = models.Sequential()
model.add(
layers.Embedding(vocab_size,
EMBEDDING_DIM,
weights=[embedding_matrix],
trainable=False,
input_length=MAXLEN))
model.add(layers.Bidirectional(layers.LSTM(128)))
model.add(layers.Dropout(0.2))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(64, activation='relu'))
model.add(layers.Dense(3, activation='softmax'))
Adam = optimizers.Adam(learning_rate=0.0001)
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
print(model.summary())
print(K.eval(model.optimizer.lr))
es = EarlyStopping(monitor='val_loss',
mode='auto',
min_delta=0,
patience=5,
restore_best_weights=True,
print(X.shape)
print(Y.shape)
print('Build model...')
words_num = tools.word_num
EMBEDDING_SIZE = 128
HIDDEN_LAYER_SIZE = 64
model = Sequential()
#model.add(layers.embeddings.Embedding(words_num, 64, input_length=26))
model.add(
layers.embeddings.Embedding(words_num,
EMBEDDING_SIZE,
input_length=input_data_X_size))
model.add(layers.LSTM(HIDDEN_LAYER_SIZE, dropout=0.1, return_sequences=True))
#model.add(layers.Dropout(0.1))
model.add(layers.LSTM(64, return_sequences=True))
#model.add(layers.Dropout(0.1))
model.add(layers.Flatten())
model.add(layers.Dense(2)) #[0, 1] or [1, 0]
model.add(layers.Activation('softmax'))
#optimizer=keras.optimizers.Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
model.fit(X, Y, epochs=1, batch_size=64, validation_split=0.05, verbose=2)
model.save(dir_path + '/log/qinggan.save.' +
def cnn_2x_siamese(voc_size, max_len, dropout=0.5):
# Determine proper input shape
#input_shape = _obtain_input_shape(input_shape,
# default_size=224,
# min_size=197,
# data_format=K.image_data_format(),
# include_top=include_top)
pivot_input = layers.Input(shape=(max_len, ), dtype='int32')
statement_input = layers.Input(shape=(max_len, ), dtype='int32')
x = layers.Embedding(output_dim=256,
input_dim=voc_size,
input_length=max_len)(pivot_input)
#if input_tensor is None:
# img_input = Input(shape=input_shape)
# else:
# if not K.is_keras_tensor(input_tensor):
# img_input = Input(tensor=input_tensor, shape=input_shape)
# else:
# img_input = input_tensor
x = ZeroPadding1D(padding=1)(x)
x = Conv1D(64, 7, strides=2, name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling1D(3, strides=2)(x)
x = conv_block(x, 3, 32, stage=2, block='a')
x = identity_block(x, 3, 32, stage=2, block='b')
x = identity_block(x, 3, 32, stage=2, block='c')
x = conv_block(x, 3, 64, stage=3, block='a')
x = identity_block(x, 3, 64, stage=3, block='b')
x = identity_block(x, 1, 64, stage=3, block='c')
x = identity_block(x, 3, 64, stage=3, block='d')
x = conv_block(x, 3, 128, stage=4, block='a')
x = identity_block(x, 3, 128, stage=4, block='b')
x = identity_block(x, 3, 128, stage=4, block='c')
x = identity_block(x, 3, 128, stage=4, block='d')
x = identity_block(x, 3, 128, stage=4, block='e')
x = identity_block(x, 3, 128, stage=4, block='f')
x = conv_block(x, 3, 256, stage=5, block='a')
x = identity_block(x, 3, 256, stage=5, block='b')
x = identity_block(x, 3, 256, stage=5, block='c')
x = AveragePooling1D(7, name='avg_pool')(x)
embedded_pivot = layers.LSTM(256)(x)
encoder_model = Model(pivot_input, embedded_pivot)
embedded_statement = encoder_model(statement_input)
concat = layers.merge([embedded_pivot, embedded_statement], mode='concat')
x = layers.Dense(256, activation='relu')(concat)
x = layers.Dropout(dropout)(x)
prediction = layers.Dense(1, activation='sigmoid')(x)
model = Model([pivot_input, statement_input], prediction)
return model
def __build_net__(self):
# 构建模型/网络
# input
inputs_data = layers.Input(shape=(
50,
75,
))
inputs_reward = layers.Input(shape=(1, ))
inputs_target = layers.Input(shape=(50, ))
# net
x = layers.LSTM(units=128,
batch_input_shape=(None, 50, 75),
return_sequences=0)(inputs_data)
x = layers.Dense(128, activation='relu')(x)
predictions = layers.Dense(50, activation='sigmoid')(x)
# model_choose
model_choose = Model(inputs=inputs_data, outputs=predictions)
# loss
def lambda_fuc_1(inputx):
predictions, inputs_target, inputs_reward = inputx
result = K.mean(
K.square(predictions - inputs_target)) * (inputs_reward)
# return K.flatten(inputs_reward+1)
return result
output_loss = layers.Lambda(function=lambda_fuc_1)(
[predictions, inputs_target, inputs_reward])
# output_loss = layers.Lambda(lambda x: ((x[0]-x[1])**2)*x[2])([predictions, inputs_target, inputs_reward])
# model_for_train
model_for_train = Model(
inputs=[inputs_data, inputs_target, inputs_reward],
outputs=[output_loss, predictions])
# summary
model_choose.summary()
model_for_train.summary()
plot_model(model_choose,
to_file="model_image/choose_model.png",
show_layer_names=True,
show_shapes=True)
plot_model(model_for_train,
to_file="model_image/choose_model_for_trian.png",
show_layer_names=True,
show_shapes=True)
# 编译步骤
def lambda_x(y_true, y_pred):
return y_pred
def lambda_y(y_true, y_pred):
result = K.zeros_like(y_pred)
return result
losses = [
lambda_x, # loss is computed in Lambda layer
lambda_y # we only include this for the metrics
]
model_choose.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model_for_train.compile(
# optimizer='rmsprop',
optimizer='rmsprop',
loss=losses,
)
return model_choose, model_for_train
#################### train and test split
x_train_lstm_daily = samples_lstm_daily[:4000]
x_test_lstm_daily = samples_lstm_daily[4000:]
y_train = labels[:4000]
y_test = labels[4000:]
print(x_train_lstm_daily.shape,y_train.shape)
print(x_test_lstm_daily.shape,y_test.shape)
plt.figure(figsize=(20,10))
plt.plot(y_test,'r')
################################################ Model: daily Block - Multi-STGCnet
input130 = Input(shape=(pre,1), dtype='float')
input131 = layers.LSTM(35,return_sequences=True)(input130)
input132 = layers.LSTM(12)(input131)
output13 = layers.Dense(1,activation='relu')(input132)
model = models.Model(inputs=[input130],outputs=[output13])
model.summary()
model.compile(optimizer='rmsprop',loss='mae',metrics=['mae','mse','mape'])
callbacks_list = [
keras.callbacks.ModelCheckpoint(filepath='lstm_daily.h5',
monitor='val_loss',
save_best_only=True,)
################################################ Model training
epochs = 1000
H = model.fit([x_train_lstm_daily], y_train,callbacks=callbacks_list,batch_size=interval_batch,epochs=epochs,validation_data=([x_test_lstm_daily],y_test))
################################################ Loss
from keras import Sequential
max_features = 10000
maxlen = 500
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = [x[::-1] for x in x_train]
x_test = [x[::-1] for x in x_test]
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
models = Sequential()
model.add(layers.Embedding(max_features, 128))
model.add(layers.LSTM(32))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binarycrossentropy', metrics=['acc'])
accuracy = model.fit(x_train,
y_train,
epochs=10,
batch_size=128,
validation_split=0.2)
model = Sequential()
model.add(layers.Embedding(max_features, 32))
model.add(layers.Bidirectional((layers.LSTM(32))))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop', loss='binarycrossentropy', metrics=['acc'])
accuracy = model.fit(x_train,
if layer.__class__.__name__ == 'GRU':
W = [np.split(w, 3, axis=-1) for w in weights]
return sum(map(list, zip(*W)), [])
elif layer.__class__.__name__ in ('LSTM', 'ConvLSTM2D'):
W = [np.split(w, 4, axis=-1) for w in weights]
for w in W:
w[2], w[1] = w[1], w[2]
return sum(map(list, zip(*W)), [])
elif layer.__class__.__name__ == 'Conv2DTranspose':
return [np.transpose(weights[0], (2, 3, 0, 1)), weights[1]]
return weights
@pytest.mark.parametrize("layer", [
layers.GRU(2, input_shape=[3, 5]),
layers.LSTM(2, input_shape=[3, 5]),
layers.ConvLSTM2D(
5, (3, 3), input_shape=[6, 6, 6, 6], data_format='channels_first'),
],
ids=['GRU', 'LSTM', 'ConvLSTM2D'])
def test_preprocess_weights_for_loading(layer):
# A model is needed to initialize weights.
_ = Sequential([layer])
weights1 = layer.get_weights()
weights2 = saving.preprocess_weights_for_loading(
layer, convert_weights(layer, weights1), original_keras_version='1')
assert all([np.allclose(x, y, 1e-5) for (x, y) in zip(weights1, weights2)])
@pytest.mark.parametrize("layer", [
layers.Conv2D(2, (3, 3), input_shape=[5, 5, 3]),
def test_sequential_regression():
# start with a basic example of using a Sequential model
# inside the functional API
seq = models.Sequential()
seq.add(layers.Dense(10, input_shape=(10, )))
x = layers.Input(shape=(10, ))
y = seq(x)
model = models.Model(x, y)
model.compile('rmsprop', 'mse')
weights = model.get_weights()
# test serialization
config = model.get_config()
model = models.Model.from_config(config)
model.compile('rmsprop', 'mse')
model.set_weights(weights)
# more advanced model with multiple branches
branch_1 = models.Sequential(name='branch_1')
branch_1.add(
layers.Embedding(input_dim=100,
output_dim=10,
input_length=2,
name='embed_1'))
branch_1.add(layers.LSTM(32, name='lstm_1'))
branch_2 = models.Sequential(name='branch_2')
branch_2.add(layers.Dense(32, input_shape=(8, ), name='dense_2'))
branch_3 = models.Sequential(name='branch_3')
branch_3.add(layers.Dense(32, input_shape=(6, ), name='dense_3'))
branch_1_2 = models.Sequential(
[legacy_layers.Merge([branch_1, branch_2], mode='concat')],
name='branch_1_2')
branch_1_2.add(layers.Dense(16, name='dense_1_2-0'))
# test whether impromptu input_shape breaks the model
branch_1_2.add(layers.Dense(16, input_shape=(16, ), name='dense_1_2-1'))
model = models.Sequential(
[legacy_layers.Merge([branch_1_2, branch_3], mode='concat')],
name='final')
model.add(layers.Dense(16, name='dense_final'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
x = (100 * np.random.random((100, 2))).astype('int32')
y = np.random.random((100, 8))
z = np.random.random((100, 6))
labels = np.random.random((100, 16))
model.fit([x, y, z], labels, epochs=1)
# test if Sequential can be called in the functional API
a = layers.Input(shape=(2, ), dtype='int32')
b = layers.Input(shape=(8, ))
c = layers.Input(shape=(6, ))
o = model([a, b, c])
outer_model = models.Model([a, b, c], o)
outer_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
outer_model.fit([x, y, z], labels, epochs=1)
# test serialization
config = outer_model.get_config()
outer_model = models.Model.from_config(config)
outer_model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
outer_model.fit([x, y, z], labels, epochs=1)
model2 = keras.Sequential([
layers.Dense(64, input_shape=(32, )),
layers.Dense(32, activation=tf.nn.relu),
layers.Dropout(0.10),
layers.Dense(32, activation=tf.nn.relu),
layers.Dense(32, activation=tf.nn.relu),
layers.Dense(12, activation=tf.nn.relu),
layers.Dense(12, activation=tf.nn.relu),
layers.Dense(1)
])
xtrcnn = np.reshape(xtr.values, (xtr.values.shape[0], 1, xtr.values.shape[1]))
ytrcnn = np.reshape(ytr.values, (ytr.shape[0], 1))
model = keras.Sequential([
layers.LSTM(64, input_shape=(xtrcnn.shape[1], xtrcnn.shape[2])),
layers.Dense(64, activation=tf.nn.relu),
layers.Dense(32, activation=tf.nn.relu),
layers.Dropout(0.10),
layers.Dense(32, activation=tf.nn.relu),
layers.Dense(32, activation=tf.nn.relu),
layers.Dense(12, activation=tf.nn.relu),
layers.Dense(12, activation=tf.nn.relu),
layers.Dense(1)
])
model2.compile(loss='mean_squared_error',
optimizer='adam',
metrics=['mean_absolute_error', 'mean_squared_error'])
model2.fit(tr_x.values, tr_y, epochs=5)
'layer_norm': self.layer_norm,
'mi': self.mi,
'zoneout_h': self.zoneout_h,
'zoneout_c': self.zoneout_c
}
base_config = super(LSTM, self).get_config()
return dict(list(base_config.items()) + list(config.items()))
# keras LSTM: OK
b = Bidirectional(
keras_layers.LSTM(256,
return_sequences=True,
kernel_regularizer=l2(1e-4),
recurrent_regularizer=l2(1e-4),
dropout=0,
recurrent_dropout=0,
activation='tanh'))
# customizer LSTM: Error
test = Bidirectional(
LSTM(256,
return_sequences=True,
kernel_regularizer=l2(1e-4),
recurrent_regularizer=l2(1e-4),
dropout=0,
recurrent_dropout=0,
zoneout_c=0,
zoneout_h=0,
mi=None,
from keras import optimizers
out = []
for k in [2, 6, 12, 24, 36, 48]:
for l in [2, 4, 8, 16, 32, 64]:
print('running at %d and %d' % (k, l))
sample, target = create_datasets(df, k)
ix = int(np.ceil(0.8 * len(df)))
sample_train, target_train = sample[0:ix], target[0:ix]
sample_test, target_test = sample[ix:], target[ix:]
model = models.Sequential()
model.add(
layers.LSTM(l,
input_shape=(sample_train.shape[1],
sample_train.shape[-1])))
model.add(layers.Dense(1, activation="sigmoid"))
model.compile(optimizer=optimizers.RMSprop(),
loss="binary_crossentropy",
metrics=['mae'])
hist = model.fit(sample_train,
target_train,
epochs=100,
batch_size=1,
verbose=2,
validation_data=(sample_test, target_test))
out.append([
k, l, hist.history['mean_absolute_error'][99],
hist.history['val_mean_absolute_error'][99]
batch_size=batch_size)
all_gen = generator(all_data,
lookback=lookback,
step=step,
min_index=0,
max_index=len(all_data),
batch_size=batch_size)
train_steps = (bound - lookback) // batch_size
val_steps = (len(all_data) - bound + 1 - lookback) // batch_size
test_steps = (len(all_data2) - lookback) // batch_size
all_steps = (len(all_data) - lookback) // batch_size
# dropout LSTM
model = models.Sequential()
model.add(layers.LSTM(64, input_shape=(None, all_data.shape[1] - 11)))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(optimizer=RMSprop(),
loss='binary_crossentropy',
metrics=['acc'])
history = model.fit_generator(train_gen,
steps_per_epoch=train_steps,
epochs=20,
validation_data=val_gen,
validation_steps=val_steps,
callbacks=callbacks_list,
verbose=0)
# generate trainset prediction results
model = models.load_model(model_path)
preds = model.predict_generator(all_gen, steps=all_steps)[:, 0]
#Set up the chars to be encoded
char_indices = dict((char, chars.index(char)) for char in chars)
#One hot encode the characters into binary arryas
print('Vectorization...')
x = np.zeros((len(sentences), maxlen, len(chars)), dtype=np.bool)
y = np.zeros((len(sentences), len(chars)), dtype=np.bool)
for i, sentence in enumerate(sentences):
for t, char in enumerate(sentence):
x[i, t, char_indices[char]] = 1
y[i, char_indices[next_chars[i]]] = 1
#Very simple model, standard/suitable configuration with only two layers, as we're only dealing with text
model = keras.models.Sequential()
#LSTM is our secret weapon here - as our iterations become more accurate, LSTM will preserve the good patterns and discard the noise
model.add(layers.LSTM(128, input_shape=(maxlen, len(chars))))
model.add(layers.Dense(len(chars), activation='softmax'))
optimizer = keras.optimizers.RMSprop(lr=0.01)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
#This 'sample' function is the core part of this program
#It sets up a probability distribution for which character to select after another, with the temperature variable being used as a
#weighting for entropy i.e. the higher the temp the less reliable and typical the character choice
#So leveragin this function, our program will try its best to crack the statistical code that lies behind Nietzsche's unique style
def sample(preds, temperature=1.0):
preds = np.asarray(preds).astype('float64')
preds = np.log(preds) / temperature
exp_preds = np.exp(preds)
# vectorizing data Xitrain, Xqtrain, Ytrain = vectorize(train_data, word2idx, word2idx_answers, max_len_instance, max_len_question) Xitest, Xqtest, Ytest = vectorize(test_data, word2idx, word2idx_answers, max_len_instance, max_len_question) # params epochs = 40 dropout_rate = 0.05 embedding_size_text = 112 embedding_size_question = 96 latent_size_text = 142 latent_size_question = 128 # model text_input = Input(shape=(max_len_instance,)) embedded_text = layers.Embedding(vocabulary_size, embedding_size_text)(text_input) encoded_text = layers.LSTM(latent_size_text)(embedded_text) encoded_text = Dropout(dropout_rate)(encoded_text) question_input = Input(shape=(max_len_question,)) embedded_question = layers.Embedding(vocabulary_size, embedding_size_question)(question_input) encoded_question = layers.LSTM(latent_size_question)(embedded_question) encoded_question = Dropout(dropout_rate)(encoded_question) concatenated = layers.concatenate([encoded_text, encoded_question], axis=-1) answer = layers.Dense(vocabulary_size_answers, activation='softmax')(concatenated) answer = Dropout(dropout_rate)(answer) model = Model([text_input, question_input], answer) model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['acc']) # training
