def creat_Model_BiLSTM_CnnAttention(sourcevocabsize, targetvocabsize, source_W, input_seq_lenth,
output_seq_lenth,
hidden_dim, emd_dim, loss='categorical_crossentropy', optimizer='rmsprop'):
# 0.8349149507609669--attention,lstm*2decoder
word_input = Input(shape=(input_seq_lenth,), dtype='int32')
word_embedding_RNN = Embedding(input_dim=sourcevocabsize + 1, output_dim=emd_dim, input_length=input_seq_lenth,
mask_zero=False, trainable=True, weights=[source_W])(word_input)
word_embedding_RNN = Dropout(0.5)(word_embedding_RNN)
BiLSTM = Bidirectional(LSTM(int(hidden_dim/2), return_sequences=True), merge_mode='concat')(word_embedding_RNN)
cnn = Conv1D(50, 3, activation='relu', strides=1, padding='same')(word_embedding_RNN)
cnn = Dropout(0.5)(cnn)
attention = Dense(1, activation='tanh')(cnn)
attention = Flatten()(attention)
attention = Activation('softmax')(attention)
attention = RepeatVector(hidden_dim)(attention)
attention = Permute([2, 1])(attention)
# apply the attention
representation = multiply([BiLSTM, attention])
# representation = Lambda(lambda xin: K.sum(xin, axis=1))(representation)
representation = BatchNormalization()(representation)
concat_LC_d = Dropout(0.5)(representation)
decodelayer1 = LSTM(50, return_sequences=False, go_backwards=True)(concat_LC_d)#!!!!!
repeat_decodelayer1 = RepeatVector(input_seq_lenth)(decodelayer1)
concat_decoder = concatenate([concat_LC_d, repeat_decodelayer1], axis=-1)#!!!!
decodelayer2 = LSTM(hidden_dim, return_sequences=True)(concat_decoder)
decodelayer = Dropout(0.5)(decodelayer2)
# decodelayer = LSTMDecoderCell(output_dim=hidden_dim, hidden_dim=hidden_dim)(concat_LC)
# decodelayer = LSTM(hidden_dim, return_sequences=True)(concat_LC)#0.8770848440899202
# decodelayer = Dropout(0.5)(decodelayer)
# TimeD = TimeDistributed(Dense(int(hidden_dim / 2)))(concat_LC_d)
TimeD = TimeDistributed(Dense(targetvocabsize + 1))(decodelayer)
# TimeD = Dropout(0.5)(TimeD)
model = Activation('softmax')(TimeD)#0.8769744561783556
# crf = CRF(targetvocabsize + 1, sparse_target=False)
# model = crf(TimeD)
Models = Model(word_input, model)
Models.compile(loss=loss, optimizer='adam', metrics=['acc'])
# Models.compile(loss=loss, optimizer=optimizers.RMSprop(lr=0.01), metrics=['acc'])
# Models.compile(loss=crf.loss_function, optimizer='adam', metrics=[crf.accuracy])
# Models.compile(loss=crf.loss_function, optimizer=optimizers.RMSprop(lr=0.005), metrics=[crf.accuracy])
return Models
def build(self):
enc_size = self.size_of_env_observation() #
argument_size = IntegerArguments.size_of_arguments #
input_enc = InputLayer(batch_input_shape=(self.batch_size, enc_size), name='input_enc')
input_arg = InputLayer(batch_input_shape=(self.batch_size, argument_size), name='input_arg')
input_prg = Embedding(input_dim=PROGRAM_VEC_SIZE, output_dim=PROGRAM_KEY_VEC_SIZE, input_length=1,
batch_input_shape=(self.batch_size, 1))
f_enc = Sequential(name='f_enc')
f_enc.add(Merge([input_enc, input_arg], mode='concat'))
f_enc.add(MaxoutDense(128, nb_feature=4))
self.f_enc = f_enc
program_embedding = Sequential(name='program_embedding')
program_embedding.add(input_prg)
f_enc_convert = Sequential(name='f_enc_convert')
f_enc_convert.add(f_enc)
f_enc_convert.add(RepeatVector(1))
f_lstm = Sequential(name='f_lstm')
f_lstm.add(Merge([program_embedding, f_enc_convert], mode='concat'))
f_lstm.add(LSTM(256, return_sequences=False, stateful=True, W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_1'))
f_lstm.add(RepeatVector(1))
f_lstm.add(LSTM(256, return_sequences=False, stateful=True, W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_2'))
f_end = Sequential(name='f_end')
f_end.add(f_lstm)
f_end.add(Dense(1, W_regularizer=l2(0.001)))
f_end.add(Activation('sigmoid', name='sigmoid_end'))
f_prog = Sequential(name='f_prog')
f_prog.add(f_lstm)
f_prog.add(Dense(PROGRAM_KEY_VEC_SIZE, activation="relu"))
f_prog.add(Dense(PROGRAM_VEC_SIZE, W_regularizer=l2(0.0001)))
f_prog.add(Activation('softmax', name='softmax_prog'))
# plot(f_prog, to_file='f_prog.png', show_shapes=True)
f_args = []
for ai in range(1, IntegerArguments.max_arg_num+1):
f_arg = Sequential(name='f_arg%s' % ai)
f_arg.add(f_lstm)
f_arg.add(Dense(IntegerArguments.depth, W_regularizer=l2(0.0001)))
f_arg.add(Activation('softmax', name='softmax_arg%s' % ai))
f_args.append(f_arg)
self.model = Model([input_enc.input, input_arg.input, input_prg.input],
[f_end.output, f_prog.output] + [fa.output for fa in f_args],
name="npi")
self.compile_model()
plot(self.model, to_file='model.png', show_shapes=True)
def g():
seq = Input(shape=(input_size, nb_chars))
z = Input(shape=(z_size, ))
z_rep = RepeatVector(input_size)(z)
seq_and_z = merge([seq, z_rep], mode='concat', concat_axis=-1)
fake_prob = sequential([
LSTM(8),
RepeatVector(output_size),
LSTM(8, return_sequences=True),
TimeDistributed(Dense(nb_chars, activation='softmax')),
])(seq_and_z)
g = Model([z, seq], [fake_prob])
return g
def attention_3d_block(
slt_api_num,
feature_dim,
name='',
):
"""
:param query: (None,D)
:param key: (None,slt_api_num,D)
:param value: (None,slt_api_num,D) 一般等于key
:return:
"""
# slt_api_num = int(key.shape[1])
# feature_dim = int(key.shape[2])
query = Input(shape=(feature_dim, ), name=name + 'query_input')
key = Input(shape=(
slt_api_num,
feature_dim,
), name=name + 'key_input')
value = Input(shape=(
slt_api_num,
feature_dim,
),
name=name + 'value_input')
Repeat_query = RepeatVector(slt_api_num)(query) # (None,slt_api_num,D)
outer_prod = Multiply()([Repeat_query, key])
sub = Subtract()([Repeat_query, key])
att_score = Concatenate(name=name + 'att_info_concate')(
[Repeat_query, key, outer_prod, sub]) # (None,slt_api_num,4*D)
a = Permute((2, 1))(att_score) # shape=(?, 4*D, slt_api_num)
a = Dense(slt_api_num, activation='softmax')(
a) # shape=(?, 4*D, slt_api_num) # 每个特征上都做softmax
a = Lambda(lambda x: K.mean(x, axis=1), name=name + 'dim_reduction')(
a) # shape=(?, slt_api_num) # 所有平均得到单个service的权重
a = RepeatVector(feature_dim)(a) # shape=(?,D,slt_api_num)
a_probs = Permute(
(2, 1), name=name + 'attention_vec')(a) # shape=(?,slt_api_num,D)
output_attention_mul = Multiply(name=name + 'attention_mul')(
[value, a_probs]) # shape=(?,slt_api_num, D)
att_result = Lambda(lambda x: tf.reduce_sum(x, axis=1))(
output_attention_mul) # (None,D)
model = Model(inputs=[query, key, value],
outputs=[att_result],
name=name + 'attBlock')
return model
def __init__(self, input_dim, output_dim, depth=2,
init='glorot_uniform', inner_init='orthogonal', forget_bias_init='one',
activation='tanh', inner_activation='hard_sigmoid',
weights=None, truncate_gradient=-1, input_length=None, hidden_state=None, batch_size=None, return_sequences = False, inner_return_sequences = False, remember_state=False,go_backwards=False, **kwargs):
nlayer = depth
self.state = []
if depth < 1:
raise Exception("Minimum depth is 1")
if depth > 1 and inner_return_sequences:
nlayer = depth*2 - 1
if weights is None:
weights = [None]*nlayer
if hidden_state is None:
hidden_state = [None]*nlayer
super(DeepLSTM, self).__init__()
def get_lstm(idim, odim, rs, reverse=False):
return lstm(input_dim=idim, output_dim=odim, init=init,
inner_init=inner_init, forget_bias_init=forget_bias_init,
activation=activation, inner_activation=inner_activation,
weights=weights[len(self.layers)], truncate_gradient=truncate_gradient,
input_length=input_length, hidden_state=hidden_state[len(self.layers)],
batch_size=batch_size, return_sequences=rs,
remember_state=remember_state, go_backwards=reverse, **kwargs)
if depth == 1:
self.add(get_lstm(input_dim, output_dim, return_sequences))
else:
lstms = []
layer = get_lstm(input_dim, output_dim, inner_return_sequences, go_backwards)
lstms.append(layer)
self.add(layer)
if not inner_return_sequences:
self.add(RepeatVector(input_length))
for i in range(depth-2):
layer = get_lstm(output_dim, output_dim, inner_return_sequences)
lstms.append(layer)
self.add(layer)
if not inner_return_sequences:
self.add(RepeatVector(input_length))
layer = get_lstm(output_dim, output_dim, return_sequences)
lstms.append(layer)
self.add(layer)
for i in range(len(lstms)-1):#connect hidden layers.
lstms[i].broadcast_state(lstms[i+1])
def model_train(X, Y, input_dim, output_dim, hidden_dim):
# input_maxlen = max(map(len,input_data_filter))
clf = Sequential()
# clf.add(Embedding(input_dim=64,output_dim=hidden_dim,input_length=input_maxlen))
# encoder
# clf.add(LSTM(hidden_dim, return_sequences=True))
clf.add(
Bidirectional(LSTM(hidden_dim, kernel_initializer='random_normal'),
input_shape=input_dim))
#decoder
clf.add(RepeatVector(input_dim[0]))
clf.add(LSTM(hidden_dim, return_sequences=True))
# clf.add(Flatten())
clf.add(TimeDistributed(Dense(output_dim)))
# clf.add(Dense(hidden_dim,activation='relu'))
clf.add(Activation('softmax'))
print('Compiling...')
time_start = time.time()
clf.compile(loss='categorical_crossentropy', optimizer='rmsprop')
time_end = time.time()
print('Compiled, cost time:%fsecond!' % (time_end - time_start))
for iter_num in range(3):
clf.fit(X, Y, batch_size=3, nb_epoch=1)
return clf
def build_model(self):
# first add input to hidden1
self.model.add(LSTM(
units=self.layers['hidden1'],
batch_input_shape=(self.batch_size,self.look_back,self.layers['input']),
#batch_size=self.batch_size,
stateful=True,
unroll=True,
return_sequences=True if self.n_hidden > 1 else False))
self.model.add(Dropout(self.dropout))
# add hidden layers
for i in range(2, self.n_hidden + 1):
return_sequences = True
if i == self.n_hidden:
return_sequences = False
self.model.add(LSTM(units = self.layers["hidden" + str(i)], stateful=True,return_sequences=return_sequences,unroll=True))
self.model.add(Dropout(self.dropout))
# add dense layer with output dimension to get output for one time_step
self.model.add(Dense(units=self.layers['output']))
# Repeat for look_ahead steps to get outputs for look_ahead timesteps.
self.model.add(RepeatVector(self.look_ahead))
# add activation
self.model.add(Activation("linear"))
# compile model and print summary
start = time.time()
self.model.compile(loss=self.loss, optimizer=Adam(lr=self.learning_rate,decay= .99))
#self.model.compile(loss=self.loss, optimizer=Adam(lr=self.learning_rate))
logging.info("Compilation Time : %s" % str(time.time() - start))
self.model.summary()
return self.model
def get_seq2seq_model_one_hot(input_size, output_size, MAX_LEN_OUTPUT):
seq2seq_model = Sequential()
seq2seq_model.add(GaussianNoise(
0.15,
input_shape=(None, input_size))) # El modelo original no tenia GN
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(
LSTM(750, return_sequences=False, activation="relu")
) # max_word_index+2 for one_hot, dim_embeddings for embedding
seq2seq_model.add(RepeatVector(MAX_LEN_OUTPUT))
seq2seq_model.add(GaussianNoise(0.15))
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(LSTM(950, return_sequences=True, activation="relu"))
seq2seq_model.add(
BatchNormalization()) # El modelo original no tenia BN
seq2seq_model.add(
TimeDistributed(Dense(output_size, activation="softmax"))
) # max_word_index+2 for one_hot, dim_embeddings for embedding
seq2seq_model.compile(optimizer='adadelta',
sample_weight_mode="temporal",
loss='categorical_crossentropy',
metrics=['accuracy'])
return seq2seq_model
def model_f(input_rnn1_shape,
input_rnn2_shape,
output_shape,
opt,
loss,
dim_rnn=[256, 256, 256],
dim_dense=[512, 512, 128],
drop=0.25,
activations=['relu', 'linear']):
inputs_rnn1 = Input(shape=input_rnn1_shape, name='rnn1_input')
lstm1 = LSTM(dim_rnn[0], activation=activations[0])(inputs_rnn1)
inputs_rnn2 = Input(shape=input_rnn2_shape, name='rnn2_input')
lstm2 = LSTM(dim_rnn[1], activation=activations[0])(inputs_rnn2)
x = concatenate([lstm1, lstm2])
x = RepeatVector(output_shape)(x)
x = LSTM(dim_rnn[2], return_sequences=True)(x)
for dim in dim_dense:
x = TimeDistributed(Dense(dim, activation=activations[0]))(x)
x = TimeDistributed(Dropout(drop))(x)
main_out = TimeDistributed(Dense(1, activation=activations[1]))(x)
main_out = Flatten()(main_out)
model = Model(inputs=[inputs_rnn1, inputs_rnn2], outputs=main_out)
model.compile(optimizer=opt, loss=loss)
return model
def _buildAutoencoderQSPR(self, z, latent_rep_size, max_length,
charset_length):
h = Dense(latent_rep_size, name='latent_input', activation='relu')(z)
h = RepeatVector(max_length, name='repeat_vector')(h)
h = GRU(501, return_sequences=True, name='gru_1')(h)
h = GRU(501, return_sequences=True, name='gru_2')(h)
h = GRU(501, return_sequences=True, name='gru_3')(h)
h2 = GRU(501, return_sequences=True, name='cation_gru_1')(h)
h2 = GRU(501, return_sequences=True, name='cation_gru_2')(h2)
h2 = GRU(501, return_sequences=True, name='cation_gru_3')(h2)
cat_smiles_decoded = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='cation_decoded_mean')(h2)
h3 = GRU(501, return_sequences=True, name='anion_gru_1')(h)
h3 = GRU(501, return_sequences=True, name='anion_gru_2')(h3)
h3 = GRU(501, return_sequences=True, name='anion_gru_3')(h3)
ani_smiles_decoded = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='anion_decoded_mean')(h3)
h = Dense(latent_rep_size * 2, name='qspr_input', activation='relu')(z)
h = Dense(100, activation='relu', name='hl_1')(h)
h = Dropout(0.5)(h)
smiles_qspr = Dense(1, activation='linear', name='qspr')(h)
return cat_smiles_decoded, ani_smiles_decoded, smiles_qspr
def build_model(preprocessor):
model = Sequential()
# encoder network
model.add(
Embedding(X_vocab_size,
X_max_len,
input_length=X_max_len,
mask_zero=True,
weights=preprocessor.init_vectors,
input_shape=(X_max_len, )))
model.add(LSTM(hidden_size))
model.add(RepeatVector(max_seq_len))
# decoder network
for _ in range(num_layers):
model.add(LSTM(hidden_size, return_sequences=True))
model.add(TimeDistributed(Dense(y_vocab_size + 1)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
return model
def construct_model(maxlen, input_dimension, output_dimension,
lstm_vector_output_dim):
"""
Склеены три слова
"""
input = Input(shape=(maxlen, input_dimension), name='input')
# lstm_encode = LSTM(lstm_vector_output_dim)(input)
lstm_encode = SimpleRNN(lstm_vector_output_dim,
activation='sigmoid')(input)
encoded_copied = RepeatVector(n=maxlen)(lstm_encode)
# lstm_decode = LSTM(output_dim=output_dimension, return_sequences=True, activation='softmax')(encoded_copied)
lstm_decode = SimpleRNN(output_dim=output_dimension,
return_sequences=True,
activation='softmax')(encoded_copied)
decoded = TimeDistributed(Dense(output_dimension,
activation='softmax'))(lstm_decode)
encoder_decoder = Model(input, decoded)
adam = Adam()
encoder_decoder.compile(loss='categorical_crossentropy', optimizer=adam)
return encoder_decoder
def _buildDecoder(self, z, latent_rep_size, max_length, charset_length, activation='relu'):
h = Dense(latent_rep_size, name='latent_input')(z)
h = RepeatVector(max_length, name='repeat_vector')(h)
h = GRU(501, return_sequences = True, name='gru_1')(h)
h = GRU(501, return_sequences = True, name='gru_2')(h)
h = GRU(501, return_sequences = True, name='gru_3')(h)
return TimeDistributed(Dense(charset_length, activation='softmax'), name='decoded_mean')(h)
def EnsembleModel():
scene_model = SceneModel()
social_model = SocialModel()
person_model = PersonModel()
scene_input = Input(shape=(img_height, img_width, img_channels))
social_input = Input(shape= (tsteps, 64))
person_input = Input(shape= (tsteps, 2))
scene_model = scene_model(scene_input)
social_model = social_model(social_input)
person_model = person_model(person_input)
input_braches = add([scene_model, social_model, person_model])
x = RepeatVector(predicting_frame_num)(input_braches)
x = GRU(128,
input_shape=(predicting_frame_num, 2),
batch_size=batch_size,
return_sequences=True,
stateful=False,
dropout=0.2)(x)
x = TimeDistributed(Dense(2))(x)
output = Activation('linear')(x)
#r = Flatten()(x)
#output2 = Dense(2, activation = 'linear')(r)
model = Model(inputs = [scene_input, social_input, person_input], outputs = output)
return model
def build_model(self):
self.model.add(
LSTM(units=self.layers['hidden1'],
batch_input_shape=(self.batch_size, self.look_back,
self.layers['input']),
stateful=True,
unroll=True,
return_sequences=True if self.n_hidden > 1 else False))
self.model.add(Dropout(self.dropout))
for i in range(2, self.n_hidden + 1):
return_sequence = True
if i == self.n_hidden:
return_sequence = False
self.model.add(
LSTM(units=self.layers['hidden' + str(i)],
stateful=True,
return_sequences=return_sequence))
self.model.add(Dense(units=self.layers['output']))
self.model.add(RepeatVector(self.look_ahead))
self.model.add(Activation('linear'))
self.model.compile(loss=self.loss,
optimizer=Adam(lr=self.learning_rate, decay=.99))
self.model.summary()
return self.model
def __build_readout_decoder__(self):
self.decoder.add(
RepeatVector(
self.sequence_len,
input_shape=(self.enc_layer_output[-1],
))) # Repeat the final vector for answer input
# Using recurrentshop's container with readout
container = RecurrentContainer(readout=True,
return_sequences=True,
output_length=self.sequence_len)
if len(self.dec_layer_output) > 1:
container.add(
LSTMCell(output_dim=self.dec_layer_output[0],
input_dim=self.enc_layer_output[-1]))
for dl in self.dec_layer_output[1:-1]:
container.add(LSTMCell(output_dim=dl))
container.add(LSTMCell(output_dim=self.enc_layer_output[-1]))
else:
container.add(
LSTMCell(input_dim=self.enc_layer_output[-1],
output_dim=self.enc_layer_output[-1]))
if self.enc_layer_output[-1] != self.dec_layer_output[-1]:
print(
'WARNING: Overriding final decoder output to %s for readout compatibility'
% self.enc_layer_output[-1])
self.decoder.add(container)
def create(self,
charset,
max_length=120,
epsilon_std=0.01,
latent_rep_size=292,
weights_file=None):
charset_length = len(charset)
x = Input(shape=(max_length, charset_length))
h = Convolution1D(9, 9, activation='relu')(x)
h = Convolution1D(9, 9, activation='relu')(h)
h = Convolution1D(10, 11, activation='relu')(h)
h = Flatten()(h)
h = Dense(435, activation='relu')(h)
z_mean = Dense(latent_rep_size, name='z_mean', activation='linear')(h)
z_log_var = Dense(latent_rep_size,
name='z_log_var',
activation='linear')(h)
def sampling(args):
z_mean, z_log_var = args
batch_size = K.shape(z_mean)[0]
epsilon = K.random_normal(shape=(batch_size, latent_rep_size),
mean=0.,
std=epsilon_std)
return z_mean + K.exp(z_log_var / 2) * epsilon
z = Lambda(sampling)([z_mean, z_log_var])
h = Dense(latent_rep_size, name='latent_input', activation='relu')(z)
h = RepeatVector(max_length)(h)
h = GRU(501, return_sequences=True)(h)
h = GRU(501, return_sequences=True)(h)
h = GRU(501, return_sequences=True)(h)
decoded_mean = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='decoded_mean')(h)
def vae_loss(x, x_decoded_mean):
x = K.flatten(x)
x_decoded_mean = K.flatten(x_decoded_mean)
xent_loss = max_length * objectives.binary_crossentropy(
x, x_decoded_mean)
kl_loss = -0.5 * K.mean(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return xent_loss + kl_loss
encoded_input = Input(shape=(max_length, latent_rep_size))
self.autoencoder = Model(x, decoded_mean)
self.encoder = Model(x, z_mean)
#self.decoder = Model(self.autoencoder.get_layer('z_mean'),
# self.autoencoder.get_layer('decoded_mean')(encoded_input))
if weights_file:
self.autoencoder.load_weights(weights_file)
self.autoencoder.compile(optimizer='Adam',
loss=vae_loss,
metrics=['accuracy'])
def prep_model(glove, dropout=0.3, l2reg=1e-3):
# XXX: this is a bit hacky to combine dot-product with six-wise output
model0 = Sequential() # s0
model1 = Sequential() # s1
model0.add(Dropout(dropout, input_shape=(glove.N,)))
model1.add(Dropout(dropout, input_shape=(glove.N,)))
model0.add(Dense(input_dim=glove.N, output_dim=glove.N, W_regularizer=l2(l2reg))) # M matrix
model0.add(RepeatVector(6)) # [nclass]
model1.add(RepeatVector(6)) # [nclass]
model = Sequential()
model.add(Merge([model0, model1], mode='dot', dot_axes=([2], [2])))
model.add(Flatten()) # 6x6 matrix with cross-activations -> 36 vector
model.add(Dense(6, W_regularizer=l2(l2reg))) # 36 vector -> 6 vector, ugh
model.add(Activation('softmax'))
return model
def load_pretrained_model(inception_wpath, colornet_wpath):
'''Load Emil's pretrained model'''
print('Loading pretrained model... (it could take a while)')
#Load weights of InceptionResNet model for embedding extraction
inception = InceptionResNetV2(weights=None, include_top=True)
inception.load_weights(inception_wpath)
inception.graph = tf.get_default_graph()
# The Model
def conv_stack(data, filters, s):
"""Utility for building conv layer"""
output = Conv2D(filters, (3, 3),
strides=s,
activation='relu',
padding='same')(data)
return output
embed_input = Input(shape=(1000, ))
#Encoder
encoder_input = Input(shape=(
256,
256,
1,
))
encoder_output = conv_stack(encoder_input, 64, 2)
encoder_output = conv_stack(encoder_output, 128, 1)
encoder_output = conv_stack(encoder_output, 128, 2)
encoder_output = conv_stack(encoder_output, 256, 1)
encoder_output = conv_stack(encoder_output, 256, 2)
encoder_output = conv_stack(encoder_output, 512, 1)
encoder_output = conv_stack(encoder_output, 512, 1)
encoder_output = conv_stack(encoder_output, 256, 1)
#Fusion
# y_mid: (None, 256, 28, 28)
fusion_output = RepeatVector(32 * 32)(embed_input)
fusion_output = Reshape(([32, 32, 1000]))(fusion_output)
fusion_output = concatenate([encoder_output, fusion_output], axis=3)
fusion_output = Conv2D(256, (1, 1), activation='relu')(fusion_output)
#Decoder
decoder_output = conv_stack(fusion_output, 128, 1)
decoder_output = UpSampling2D((2, 2))(decoder_output)
decoder_output = conv_stack(decoder_output, 64, 1)
decoder_output = UpSampling2D((2, 2))(decoder_output)
decoder_output = conv_stack(decoder_output, 32, 1)
decoder_output = conv_stack(decoder_output, 16, 1)
decoder_output = Conv2D(2, (2, 2), activation='tanh',
padding='same')(decoder_output)
decoder_output = UpSampling2D((2, 2))(decoder_output)
model = Model(inputs=[encoder_input, embed_input], outputs=decoder_output)
# Load colornet weights
model.load_weights(colornet_wpath)
print('Model loaded!')
return (model, inception)
def train():
print('Loading dataset: {} ...'.format(WAV_FILE))
samples, _ = libcore.load(WAV_FILE, sr=SAMPLING_RATE)
power = np.mean(samples ** 2) * 0.5
print('Sampling training set nb_samples={}, size=({},{}) ...'.format(TS_SIZE, SEQ_LEN, INPUT_DIM))
training_set = np.array([sample_chunk(samples, power).T for _ in range(TS_SIZE)])
print('Constructing autoencoder ...')
inputs = Input(shape=(SEQ_LEN, INPUT_DIM))
enc_1 = GRU(128)(inputs)
features = RepeatVector(SEQ_LEN)(enc_1)
dec_0 = GRU(128, return_sequences=True)(features)
dec_1 = GRU(INPUT_DIM, return_sequences=True)(dec_0)
autoencoder = Model(inputs, dec_1)
autoencoder.summary()
autoencoder.compile(optimizer='rmsprop', loss='mse')
model_cb = ModelCheckpoint(WEIGHT_FILE_PATTERN, monitor='val_loss', verbose=0,
save_best_only=False, save_weights_only=False, mode='auto', period=SAVE_AFTER)
print('Training autoencoder for {} epochs. Save each {}th epoch ...'.format(T_EPOCHS, SAVE_AFTER))
history = autoencoder.fit(training_set, training_set, nb_epoch=T_EPOCHS, validation_split=0.1, callbacks=[model_cb])
def __init__(self,
output_dim,
hidden_dim,
output_length,
depth=1,
dropout=0.25,
**kwargs):
super(SimpleSeq2seq, self).__init__()
if type(depth) not in [list, tuple]:
depth = (depth, depth)
self.encoder = LSTM(hidden_dim, **kwargs)
self.decoder = LSTM(hidden_dim if depth[1] > 1 else output_dim,
return_sequences=True,
**kwargs)
for i in range(1, depth[0]):
self.add(LSTM(hidden_dim, return_sequences=True, **kwargs))
self.add(Dropout(dropout))
self.add(self.encoder)
self.add(Dropout(dropout))
self.add(RepeatVector(output_length))
self.add(self.decoder)
for i in range(1, depth[1]):
self.add(LSTM(hidden_dim, return_sequences=True, **kwargs))
self.add(Dropout(dropout))
if depth[1] > 1:
self.add(TimeDistributedDense(output_dim))
def base_model(feature_len=1, after_day=1, input_shape=(20, 1)):
# 序列模型是一个线性的层次堆栈
model = Sequential()
# 1维卷积层，用于输入一维输入信号进行领域滤波
model.add(
Conv1D(10,
kernel_size=5,
input_shape=input_shape,
activation='relu',
padding='valid',
strides=1))
# 循环层
model.add(LSTM(100, return_sequences=False, input_shape=input_shape))
# Dropout层，防止过拟合
model.add(Dropout(0.25))
# one to many
# RepeatVector层将输入重复n次
model.add(RepeatVector(after_day))
model.add(LSTM(200, return_sequences=True))
model.add(Dropout(0.25))
# 包装器，把一层应用到每一个时间步上
# Dense 全连接层
model.add(
TimeDistributed(
Dense(100, activation='relu', kernel_initializer='uniform')))
model.add(
TimeDistributed(
Dense(feature_len,
activation='linear',
kernel_initializer='uniform')))
return model
def _buildDecoder(self,
z,
latent_rep_size,
max_length,
charset_length,
gru_layers=3,
gru_output_units=501,
mol_inputs=1):
h = Dense(latent_rep_size, name='latent_input', activation='relu')(z)
h = RepeatVector(max_length, name='repeat_vector')(h)
h = GRU(501, return_sequences=True, name='gru_1')(h)
for gru in range(gru_layers - 2):
h = GRU(gru_output_units,
return_sequences=True,
name='gru_{}'.format(gru + 2))(h)
h = GRU(501, return_sequences=True,
name='gru_{}'.format(gru_layers))(h)
if mol_inputs == 1:
smiles_decoded = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='decoded_mean')(h)
elif mol_inputs == 2:
cation_decoded = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='decoded_cation_mean')(h)
anion_decoded = TimeDistributed(Dense(charset_length,
activation='softmax'),
name='decoded_anion_mean')(h)
if mol_inputs == 1:
return smiles_decoded
elif mol_inputs == 2:
return cation_decoded, anion_decoded
def __call__(self,
vocabulary_size=5000,
maxlen=100,
embedding_dim=300,
filter_length=[3],
layer=-1,
hidden_dim=250,
nb_class=2,
drop_out_prob=0.5,
use_my_embedding=False,
embedding_weights=None):
filter_row = filter_length[0]
filter_col = 1
nb_filter = embedding_dim
self.log_params(locals())
model = Sequential()
maxlen = self.add_embedding(model, vocabulary_size, embedding_dim,
maxlen, use_my_embedding,
embedding_weights)
model.add(Permute((2, 1)))
model.add(Reshape((nb_filter * maxlen, )))
model.add(RepeatVector(filter_col))
model.add(Permute((2, 1)))
model.add(Reshape((nb_filter, maxlen, filter_col)))
max_possible_layer = int(math.log(maxlen, 2)) - 1
if layer > 0:
assert layer < max_possible_layer
else:
layer = max_possible_layer
for i in range(layer):
# model.add(Dropout(drop_out_prob/(i+1)))
model.add(
Convolution2D(nb_filter=nb_filter,
nb_row=filter_row,
nb_col=1,
border_mode='same',
activation='relu',
dim_ordering='th',
subsample=(1, 1)))
model.add(MaxPooling2D(pool_size=(2, 1)))
# If max_possible_layer is adopted, add one more layer
if i == max_possible_layer - 1:
logging.debug("Last layer added")
model.add(
Convolution2D(nb_filter=nb_filter,
nb_row=2,
nb_col=1,
border_mode='valid',
activation='relu',
dim_ordering='th',
subsample=(1, 1)))
self.add_full(model, hidden_dim, drop_out_prob, nb_class)
return model
def create_model_bidirectional(vocab_len,
batch_size,
hidden_units=HIDDEN_UNITS,
depth=1,
impl=0):
main_input = Input(shape=(MAX_SEQ_LEN, ), dtype='int32', name='main_input')
word_embedding = Embedding(input_dim=vocab_len,
output_dim=batch_size,
input_length=MAX_SEQ_LEN,
mask_zero=True)(main_input)
encoder = GRU(units=hidden_units,
implementation=impl,
return_sequences=False)(word_embedding)
expand_encoder = RepeatVector(n=MAX_SEQ_LEN)(encoder)
forward_decoder = GRU(units=hidden_units,
implementation=impl,
return_sequences=True)(expand_encoder)
reverse_decoder = GRU(units=hidden_units,
implementation=impl,
return_sequences=True)(expand_encoder)
forward_softmax = Activation('softmax')(TimeDistributed(
Dense(vocab_len))(forward_decoder))
reverse_softmax = Activation('softmax')(TimeDistributed(
Dense(vocab_len))(reverse_decoder))
model = Model(inputs=[main_input],
outputs=[forward_softmax, reverse_softmax])
return model
def VGG19_hieratt(query_in_size, query_embed_size, nb_classes):
"""Stack hierarchical attention on pre-trained VGG19.
Requires https://github.com/fchollet/deep-learning-models"""
base_model = VGG19(weights='imagenet')
input_image = base_model.input
input_question = Input(shape=(query_in_size, )) # question vector
# Model up to 3rd block
f_1 = Model(input=img_in,
output=base_model.get_layer('block3_pool').output)
f_1 = f_1(img_in)
f_1 = Reshape((256, 28 * 28))(f_1)
f_1 = Permute((2, 1))(f_1)
q_1 = Dense(query_embed_size,
activation='relu')(input_question) # Encode question
# Add question embedding to each feature column
q_1 = RepeatVector(28 * 28)(q_1)
q_f = merge([f_1, q_1], 'concat')
# Estimate and apply attention per feature
att_1 = TimeDistributedDense(1, activation="sigmoid")(q_f)
att_1 = Lambda(repeat_1, output_shape=(28 * 28, 256))(att_1)
att_1 = merge([f_1, att_1], 'mul')
# Reshape to the original feature map from previous layer
att_1 = Permute((2, 1))(att_1)
f_1_att = Reshape((256, 28, 28))(att_1)
model = Model(input=[img_in, input_question], output=f_1_att)
print model.summary()
def build_model(self):
""" Create a Keras seq2seq VAE model for sketch-rnn """
# Arrange inputs:
self.encoder_input = Input(shape=(self.hps['max_seq_len'], 5), name='encoder_input')
decoder_input = Input(shape=(self.hps['max_seq_len'], 5), name='decoder_input')
# Set recurrent dropout to fraction of units to *drop*, if in CuDNN accelerated mode, don't use dropout:
recurrent_dropout = 1.0-self.hps['recurrent_dropout_prob'] if \
(self.hps['use_recurrent_dropout'] and (self.hps['accelerate_LSTM'] is False)) else 0
# Option to use the accelerated version of LSTM, CuDNN LSTM. Much faster, but no support for recurrent dropout:
if self.hps['accelerate_LSTM'] and self.hps['is_training']:
lstm_layer_encoder = CuDNNLSTM(units=self.hps['enc_rnn_size'])
lstm_layer_decoder = CuDNNLSTM(units=self.hps['dec_rnn_size'], return_sequences=True, return_state=True)
self.hps['use_recurrent_dropout'] = False
print('Using CuDNNLSTM - No Recurrent Dropout!')
else:
# Note that in inference LSTM is always selected (even in accelerated mode) so inference on CPU is supported
lstm_layer_encoder = LSTM(units=self.hps['enc_rnn_size'], recurrent_dropout=recurrent_dropout)
lstm_layer_decoder = LSTM(units=self.hps['dec_rnn_size'], recurrent_dropout=recurrent_dropout,
return_sequences=True, return_state=True)
# Encoder, bidirectional LSTM:
encoder = Bidirectional(lstm_layer_encoder, merge_mode='concat')(self.encoder_input)
# Latent vector - [batch_size]X[z_size]:
self.batch_z = self.latent_z(encoder)
# Decoder LSTM:
self.decoder = lstm_layer_decoder
# Initial state for decoder:
self.initial_state = Dense(units=2*self.decoder.units, activation='tanh', name='dec_initial_state',
kernel_initializer=RandomNormal(mean=0.0, stddev=0.001))
initial_state = self.initial_state(self.batch_z)
# Split to hidden state and cell state:
init_h, init_c = (initial_state[:, :self.decoder.units], initial_state[:, self.decoder.units:])
# Concatenate latent vector to expected output and feed this as input to decoder:
tile_z = RepeatVector(self.hps['max_seq_len'])(self.batch_z)
decoder_full_input = Concatenate()([decoder_input, tile_z])
# Retrieve decoder output tensors:
[decoder_output, final_state1, final_state_2] = self.decoder(decoder_full_input, initial_state=[init_h, init_c])
# self.final_state = [final_state1, final_state_2] todo: not used, remove when stable
# Number of outputs:
# 3 pen state logits, 6 outputs per mixture model(mean_x, mean_y, std_x, std_y, corr_xy, mixture weight pi)
n_out = (3 + self.hps['num_mixture'] * 6)
# Output FC layer
self.output = Dense(n_out, name='output')
output = self.output(decoder_output)
# Build Keras model
model_o = Model([self.encoder_input, decoder_input], output)
return model_o
def rel_types_model(model,
ins,
max_len,
embedding_dim,
rel_types2id_size,
focus,
pre='rtypes'):
"""Discourse relation types model as Keras Graph."""
# prepare focus dimensionality
model.add_node(RepeatVector(rel_types2id_size),
name=pre + '_focus_rep',
input=focus)
model.add_node(Permute((2, 1)),
name=pre + '_focus',
input=pre + '_focus_rep')
# discourse relation types dense neural network (sample, time_pad, rel_types2id)
model.add_node(TimeDistributedDense(rel_types2id_size, init='he_uniform'),
name=pre + '_dense',
input=ins[0])
model.add_node(Activation('softmax'),
name=pre + '_softmax',
input=pre + '_dense')
# multiplication to focus the activations (doc, time_pad, rel_types2id)
model.add_node(Activation('linear'),
name=pre + '_out',
inputs=[pre + '_focus', pre + '_softmax'],
merge_mode='mul')
return pre + '_out'
def answer_selection(M_r, u, A, pfx='anssel'):
# input shape
# M_r: (None, c, mem_dim, 3)
# u: (None, mem_dim)
# A: [(None, mem_dim), (None, mem_dim), (None, mem_dim), (None, mem_dim), (None, mem_dim)]
c = M_r._keras_shape[1]
u_ = q_tile(c, pfx)(u) # shape = (None, c, mem_dim, 3)
dot_prod1 = Lambda(
name='dot_prod1_' + pfx,
function=lambda x: K.sum(multiply([x[0], x[1]]), axis=2))
p = dot_prod1([M_r, u_]) # shape = (None, c, 3)
p = Activation('softmax')(Reshape((c * 3, ))(p)) # shape = (None, c*3)
p = RepeatVector(conf['mem_dim'])(p) # shape = (None, mem_dim, c*3)
permute = Lambda(name='permute_' + pfx,
function=lambda x: K.permute_dimensions(x, (0, 2, 1, 3)),
output_shape=lambda shape: (shape[0], ) + (shape[2], ) +
(shape[1], ) + (shape[3], ))
u_ = permute(u_) # shape = (None, mem_dim, c, 3)
u_ = Reshape((conf['mem_dim'], c * 3))(u_) # shape = (None, mem_dim, c*3)
dot_prod2 = Lambda(
name='dot_prod2_' + pfx,
function=lambda x: K.sum(multiply([x[0], x[1]]), axis=2))
o = dot_prod2([p, u_]) # shape = (None, mem_dim)
z = score_function(pfx)([o, u, A[0], A[1], A[2], A[3], A[4]])
return z
def create(self):
language_model = Sequential()
self.textual_embedding(language_model, mask_zero=True)
self.language_model = language_model
visual_model_factory = \
select_sequential_visual_model[self._config.trainable_perception_name](
self._config.visual_dim)
visual_model = visual_model_factory.create()
visual_dimensionality = visual_model_factory.get_dimensionality()
self.visual_embedding(visual_model, visual_dimensionality)
#visual_model = Sequential()
#self.visual_embedding(visual_model)
self.visual_model = visual_model
visual_model.add(RepeatVector(self._config.max_input_time_steps))
if self._config.multimodal_merge_mode == 'dot':
self.add(
Merge([language_model, visual_model],
mode='dot',
dot_axes=[(1, ), (1, )]))
else:
self.add(
Merge([language_model, visual_model],
mode=self._config.multimodal_merge_mode))
self.add(
self._config.recurrent_encoder(
self._config.hidden_state_dim,
return_sequences=False,
go_backwards=self._config.go_backwards))
self.deep_mlp()
self.add(Dense(self._config.output_dim))
self.add(Activation('softmax'))
