OxySat_input, Platelets_input, RBC_input, pH_input, Hematocrit_input, ResRate_input, Weight_input,
HADM_input, Blood_input, Digestive_input, Genitourinary_input, Illdefined_input, Infectious_input,
Injury_input, Nervous_input, Respiratory_input, Skin_input, Marriage_input, Religion_input,
Insurance_input, AdmissionArea_input, AdmissionType_input])
layerone = LSTM(cells, return_sequences=True)(merge_one)
layerone = Dropout(0.3)(layerone)
layertwo = LSTM(cells, return_sequences=True)(layerone)
layertwo = Dropout(0.3)(layertwo)
merge_two = Add()([layerone, layertwo])
layerthree = LSTM(cells, return_sequences=True)(merge_two)
layerthree = Dropout(0.3)(layerthree)
merge_three = Add()([layerone, layertwo, layerthree])
layerfour = LSTM(cells, return_sequences=True)(merge_three)
layerfour = Dropout(0.3)(layerfour)
out = TimeDistributed(Dense(1, activation='relu'))(layerfour)
model = Model(inputs=[inpHR,inpTemp, inpWBC, inpGlas, inpMech_Vent, inpBiril, inpBUN, inpAlbumin, inpOxySat, inpPlatelets, inpRBC, inppH,
inpHematocrit, inpResRate, inpWeight, inpHADM,
inpBlood, inpDigestive, inpGenitourinary, inpIlldefined, inpInfectious, inpInjury, inpNervous, inpRespiratory, inpSkin, inpMarriage,
inpReligion, inpInsurance, inpAdmissionArea, inpAdmissionType],
outputs=out)
model.compile(optimizer= opt , loss= "mean_squared_logarithmic_error" ,
metrics=['mean_absolute_error', 'mse' ])
history = model.fit(x_train, y_train,epochs=epochs, batch_size = batch_size, verbose=2)
print("Evaluate on test data")
results = model.evaluate(x_test, y_test, batch_size = batch_size)
print("MSLE, MAE, MSE:", results)
print("%s: %.2f%%" % (model.metrics_names[0], results[0]))
cvscores.append(results[0])
inp = Input( shape=( MAX_SEQ_LEN,) )
emb = Embedding( input_dim=VOCAB_SIZE, output_dim=50, weights=[embedding_matrix],
trainable=False, input_length=MAX_SEQ_LEN)(inp)
embedding = Model( inp, emb )
save_model( embedding, "embedding_model.h5" )
#Model for identifying tags of each word
inp = Input( shape=(MAX_SEQ_LEN, 50) )
drop = Dropout(0.1)(inp)
#two bidirectinal LSTM layers
lstm1 = LSTM( 50, return_sequences=True, recurrent_dropout=0.1)
seq1 = Bidirectional(lstm1)( drop )
lstm2 = LSTM( 50, return_sequences=True, recurrent_dropout=0.1)
seq2 = Bidirectional(lstm2)( seq1 )
# TIME_DISTRIBUTED -> ( MAX_SEQ_LEN, 50 ) -> (MAX_SEQ_LEN, POS_SIZE)
tags = TimeDistributed( Dense(POS_SIZE, activation="relu") )(seq2)
model = Model( inp, tags )
model.compile( optimizer="rmsprop", loss="categorical_crossentropy", metrics=[ "accuracy" ] )
#batch generator for model training
def getBatch(sentences, targets, batch_size=128):
n = len(sentences)//batch_size
for i in range( n+1 ):
x = sentences[ i*batch_size : (i+1)*batch_size ]
x = embedding.predict(x)
y = targets[ i*batch_size : (i+1)*batch_size ]
yield x,y
#do training
all_metrics = []
for epoch in range(1,101):
def line_lstm_ctc(input_shape,
output_shape,
window_width=28,
window_stride=14):
image_height, image_width = input_shape
output_length, num_classes = output_shape
num_windows = int((image_width - window_width) / window_stride) + 1
if num_windows < output_length:
raise ValueError(
f'Window width/stride need to generate at least {output_length} windows (currently {num_windows})'
)
image_input = Input(shape=input_shape, name='image')
y_true = Input(shape=(output_length, ), name='y_true')
input_length = Input(shape=(1, ), name='input_length')
label_length = Input(shape=(1, ), name='label_length')
gpu_present = len(device_lib.list_local_devices()) > 1
lstm_fn = CuDNNLSTM if gpu_present else LSTM
# Your code should use slide_window and extract image patches from image_input.
# Pass a convolutional model over each image patch to generate a feature vector per window.
# Pass these features through one or more LSTM layers.
# Convert the lstm outputs to softmax outputs.
# Note that lstms expect a input of shape (num_batch_size, num_timesteps, feature_length).
##### Your code below (Lab 3)
image_reshaped = Reshape((image_height, image_width, 1))(image_input)
# (image_height, image_width, 1)
image_patches = Lambda(slide_window,
arguments={
'window_width': window_width,
'window_stride': window_stride
})(image_reshaped)
# (num_windows, image_height, window_width, 1)
# Make a LeNet and get rid of the last two layers (softmax and dropout)
convnet = lenet((image_height, window_width, 1), (num_classes, ))
convnet = KerasModel(inputs=convnet.inputs,
outputs=convnet.layers[-2].output)
convnet_outputs = TimeDistributed(convnet)(image_patches)
# (num_windows, 128)
lstm_output = Bidirectional(lstm_fn(
256, return_sequences=True))(convnet_outputs)
# add additional layer
lstm_output = Bidirectional(lstm_fn(128,
return_sequences=True))(lstm_output)
# (num_windows, 128)
softmax_output = Dense(num_classes,
activation='softmax',
name='softmax_output')(lstm_output)
# (num_windows, num_classes)
##### Your code above (Lab 3)
input_length_processed = Lambda(
lambda x, num_windows=None: x * num_windows,
arguments={'num_windows': num_windows})(input_length)
ctc_loss_output = Lambda(
lambda x: K.ctc_batch_cost(x[0], x[1], x[2], x[3]), name='ctc_loss')(
[y_true, softmax_output, input_length_processed, label_length])
ctc_decoded_output = Lambda(
lambda x: ctc_decode(x[0], x[1], output_length),
name='ctc_decoded')([softmax_output, input_length_processed])
model = KerasModel(
inputs=[image_input, y_true, input_length, label_length],
outputs=[ctc_loss_output, ctc_decoded_output])
return model
factor_train = np.load('train.npz')['factor']
demand_aux_train = np.load('train.npz')['auxiliary'][:, :, :, :1]
supply_aux_train = np.load('train.npz')['auxiliary'][:, :, :, 1:]
[demandX_test, supplyX_test] = np.load('test.npz')['X']
[demandY_test, supplyY_test] = np.load('test.npz')['Y']
factor_test = np.load('test.npz')['factor']
demand_aux_test = np.load('test.npz')['auxiliary'][:, :, :, :1]
supply_aux_test = np.load('test.npz')['auxiliary'][:, :, :, 1:]
timestep = 3
size = 16
dim = 4 * 4 * 16
input_demand = Input(shape=(None, size, size, 1))
demand_encoder = encoder(input_demand)
demand_reshape = TimeDistributed(Dropout(0.3))(demand_encoder)
demand_reshape = TimeDistributed(Flatten())(demand_reshape)
demand_reshape = Reshape((timestep, dim))(demand_reshape)
input_supply = Input(shape=(None, size, size, 1))
supply_encoder = encoder(input_supply)
supply_reshape = TimeDistributed(Dropout(0.3))(supply_encoder)
supply_reshape = TimeDistributed(Flatten())(supply_reshape)
supply_reshape = Reshape((timestep, dim))(supply_reshape)
"""
demand_decoder = TimeDistributed(Conv2D(16, (3, 3), padding='same', activation='relu'))(demand_encoder)
demand_decoder = TimeDistributed(UpSampling2D((2, 2)))(demand_decoder)
demand_decoder = TimeDistributed(Conv2D(8, (3, 3), padding='same', activation='relu'))(demand_decoder)
demand_decoder = TimeDistributed(UpSampling2D((2, 2)))(demand_decoder)
demand_decoder = TimeDistributed(Conv2D(1, (3, 3), padding='same', activation='relu'))(demand_decoder)
def build(self,
word_length,
target_label_dims,
word_vocab_size,
char_vocab_size,
word_embedding_dims=100,
char_embedding_dims=16,
word_lstm_dims=20,
tagger_lstm_dims=200,
dropout=0.5,
crf_mode='pad'):
"""
Build a NERCRF model
Args:
word_length (int): max word length in characters
target_label_dims (int): number of entity labels (for classification)
word_vocab_size (int): word vocabulary size
char_vocab_size (int): character vocabulary size
word_embedding_dims (int): word embedding dimensions
char_embedding_dims (int): character embedding dimensions
word_lstm_dims (int): character LSTM feature extractor output dimensions
tagger_lstm_dims (int): word tagger LSTM output dimensions
dropout (float): dropout rate
crf_mode (string): CRF operation mode, select 'pad'/'reg' for supplied sequences in
input or full sequence tagging. ('reg' is forced when use_cudnn=True)
"""
self.word_length = word_length
self.target_label_dims = target_label_dims
self.word_vocab_size = word_vocab_size
self.char_vocab_size = char_vocab_size
self.word_embedding_dims = word_embedding_dims
self.char_embedding_dims = char_embedding_dims
self.word_lstm_dims = word_lstm_dims
self.tagger_lstm_dims = tagger_lstm_dims
self.dropout = dropout
self.crf_mode = crf_mode
assert crf_mode in ('pad', 'reg'), 'crf_mode is invalid'
# build word input
words_input = Input(shape=(None, ), name='words_input')
embedding_layer = Embedding(self.word_vocab_size,
self.word_embedding_dims,
name='word_embedding')
word_embeddings = embedding_layer(words_input)
# create word character embeddings
word_chars_input = Input(shape=(None, self.word_length),
name='word_chars_input')
char_embedding_layer = Embedding(
self.char_vocab_size,
self.char_embedding_dims,
name='char_embedding')(word_chars_input)
char_embeddings = TimeDistributed(
Conv1D(128, 3, padding='same',
activation='relu'))(char_embedding_layer)
char_embeddings = TimeDistributed(
GlobalMaxPooling1D())(char_embeddings)
# create the final feature vectors
features = concatenate([word_embeddings, char_embeddings], axis=-1)
# encode using a bi-LSTM
features = Dropout(self.dropout)(features)
bilstm = Bidirectional(
self._rnn_cell(self.tagger_lstm_dims,
return_sequences=True))(features)
bilstm = Bidirectional(
self._rnn_cell(self.tagger_lstm_dims,
return_sequences=True))(bilstm)
bilstm = Dropout(self.dropout)(bilstm)
bilstm = Dense(self.target_label_dims)(bilstm)
inputs = [words_input, word_chars_input]
if self.use_cudnn:
self.crf_mode = 'reg'
with tf.device('/cpu:0'):
crf = CRF(self.target_label_dims,
mode=self.crf_mode,
name='ner_crf')
if self.crf_mode == 'pad':
sequence_lengths = Input(batch_shape=(None, 1), dtype='int32')
predictions = crf([bilstm, sequence_lengths])
inputs.append(sequence_lengths)
else:
predictions = crf(bilstm)
# compile the model
model = tf.keras.Model(inputs=inputs, outputs=predictions)
model.compile(loss={'ner_crf': crf.loss},
optimizer=tf.keras.optimizers.Adam(0.001, clipnorm=5.),
metrics=[crf.viterbi_accuracy])
self.model = model
return_state=True,
dropout=0.4,
recurrent_dropout=0.2)
decoder_outputs, decoder_fwd_state, decoder_back_state = decoder_lstm(
dec_emb, initial_state=[state_h, state_c])
# Attention layer
attn_layer = AttentionLayer(name='attention_layer')
attn_out, attn_states = attn_layer([encoder_outputs, decoder_outputs])
# Concat attention input and decoder LSTM output
decoder_concat_input = Concatenate(
axis=-1, name='concat_layer')([decoder_outputs, attn_out])
# dense layer
decoder_dense = TimeDistributed(Dense(y_voc, activation='softmax'))
decoder_outputs = decoder_dense(decoder_concat_input)
# Define the model
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
model.summary()
model.compile(optimizer='rmsprop', loss='sparse_categorical_crossentropy')
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=2)
start_train = time.time()
history = model.fit([x_tr, y_tr[:, :-1]],
y_tr.reshape(y_tr.shape[0], y_tr.shape[1], 1)[:, 1:],
epochs=50,
callbacks=[es],
# print("y_train.shape={}".format(y_train.shape))
S_inputs = Input(shape=(max_length,), dtype='int32')
# print(K.int_shape(S_inputs))
embeddings = Embedding(word_num, emb_size)(S_inputs)
# print(K.int_shape(embeddings))
lstm_seq = Bidirectional(LSTM(128,return_sequences = True))(embeddings)
lstm_seq = Position_Embedding()(lstm_seq)
# print(K.int_shape(lstm_seq))
O_seq = Attention(8, 16)([lstm_seq, lstm_seq, lstm_seq])
O_seq = Attention(8, 16)([O_seq, O_seq, O_seq])
# print(K.int_shape(O_seq))
# O_seq = GlobalAveragePooling1D()(O_seq)
# print(K.int_shape(O_seq))
# O_seq = Dropout(0.5)(O_seq)
# outputs = Dense(1, activation='sigmoid')(O_seq)
# print(K.int_shape(outputs))
outputs = TimeDistributed(Dense(class_num, activation='sigmoid'))(O_seq)
# print(K.int_shape(outputs))
model = Model(inputs=S_inputs, outputs=outputs)
print(model.summary())
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
print('Train...')
model.fit(x_train, y_train,
batch_size=batch_size,
epochs=5,
validation_data=(x_val, y_val))
score, acc = model.evaluate(x_val, y_val)
print('Test score:', score)
print('Test accuracy:', acc)
X = pad_sequences(maxlen=max_len, sequences=X, padding="post", value=num_words - 1)
y = [[tag2idx[w[2]] for w in s] for s in sentences]
y = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tag2idx["O"])
# + colab={} colab_type="code" id="q7VfnnkXpkfS"
x_train, x_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
# + colab={"base_uri": "https://localhost:8080/", "height": 330} colab_type="code" id="Aee3mCZ3pkkv" outputId="b7fb911b-21d1-43e6-adc9-bb2d8bdfb921"
input_word = Input(shape=(max_len,))
model = Embedding(input_dim=num_words, output_dim=50, input_length=max_len)(input_word)
model = SpatialDropout1D(0.1)(model)
model = Bidirectional(LSTM(units=100, return_sequences=True, recurrent_dropout=0.1))(
model
)
out = TimeDistributed(Dense(num_tags, activation="softmax"))(model)
model = Model(input_word, out)
model.summary()
model.compile(
optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"]
)
chkpt = ModelCheckpoint(
"model_weights.h5",
monitor="val_loss",
verbose=1,
save_best_only=True,
save_weights_only=True,
mode="min",
def _create_keras_model(self):
# Each input sample consists of a bag of x`MAX_CONTEXTS` tuples (source_terminal, path, target_terminal).
# The valid mask indicates for each context whether it actually exists or it is just a padding.
path_source_token_input = Input((self.config.MAX_CONTEXTS, ),
dtype=tf.int32)
path_input = Input((self.config.MAX_CONTEXTS, ), dtype=tf.int32)
path_target_token_input = Input((self.config.MAX_CONTEXTS, ),
dtype=tf.int32)
context_valid_mask = Input((self.config.MAX_CONTEXTS, ))
# Input paths are indexes, we embed these here.
paths_embedded = Embedding(self.vocabs.path_vocab.size,
self.config.PATH_EMBEDDINGS_SIZE,
name='path_embedding')(path_input)
# Input terminals are indexes, we embed these here.
token_embedding_shared_layer = Embedding(
self.vocabs.token_vocab.size,
self.config.TOKEN_EMBEDDINGS_SIZE,
name='token_embedding')
path_source_token_embedded = token_embedding_shared_layer(
path_source_token_input)
path_target_token_embedded = token_embedding_shared_layer(
path_target_token_input)
# `Context` is a concatenation of the 2 terminals & path embedding.
# Each context is a vector of size 3 * EMBEDDINGS_SIZE.
context_embedded = Concatenate()([
path_source_token_embedded, paths_embedded,
path_target_token_embedded
])
context_embedded = Dropout(1 - self.config.DROPOUT_KEEP_RATE)(
context_embedded)
# Lets get dense: Apply a dense layer for each context vector (using same weights for all of the context).
context_after_dense = TimeDistributed(
Dense(self.config.CODE_VECTOR_SIZE,
use_bias=False,
activation='tanh'))(context_embedded)
# The final code vectors are received by applying attention to the "densed" context vectors.
code_vectors, attention_weights = AttentionLayer(name='attention')(
[context_after_dense, context_valid_mask])
# "Decode": Now we use another dense layer to get the target word embedding from each code vector.
#Esta es la que da la probabilidad de que sea o no un determinado nombre. Por eso el primer parametro es la cantidad de nombres distintos de metodos que hay en el vocabulario.
#target_index = Dense(self.vocabs.target_vocab.size, use_bias=False, activation='softmax', name='target_index')(code_vectors)
#Ideally this should be our definition: target_index = Dense(1, use_bias=False, activation='sigmoid', name='target_index')(code_vectors)
target_index = Dense(1,
use_bias=False,
activation='sigmoid',
name='target_index')(code_vectors)
# Wrap the layers into a Keras model, using our subtoken-metrics and the CE loss.
inputs = [
path_source_token_input, path_input, path_target_token_input,
context_valid_mask
]
self.keras_train_model = keras.Model(inputs=inputs,
outputs=target_index)
keras.utils.plot_model(self.keras_train_model,
'code2vecTrainModel.png',
show_shapes=True)
def build_action_net(self, input_tensor, action_dim):
output_layer = Dense(action_dim, activation=None)
output_tensor = TimeDistributed(output_layer)(input_tensor)
return output_tensor
series5 = generateTimeSeries(10000, nSteps + stepsAhead)
XTrain5 = series5[:7000, :nSteps]
XValid5 = series5[7000:9000, :nSteps]
XTest5 = series5[9000:, :nSteps]
yTrain5 = np.empty((10000, nSteps, stepsAhead))
for s in range(1, stepsAhead + 1):
yTrain5[:, :, s - 1] = series5[:, s:s + nSteps, 0]
yValid5 = yTrain5[7000:9000]
yTest5 = yTrain5[9000:]
yTrain5 = yTrain5[:7000]
from tensorflow.keras.layers import TimeDistributed
inputLayer5 = Input(shape=[None, 1])
tmp = SimpleRNN(20, return_sequences=True)(inputLayer5)
tmp = SimpleRNN(20, return_sequences=True)(tmp)
outputLayer5 = TimeDistributed(Dense(10))(tmp)
model5 = Model(inputLayer5, outputLayer5)
model5.compile(optimizer='rmsprop', loss='mse')
model5.summary()
history5 = model5.fit(XTrain5,
yTrain5,
epochs=20,
validation_data=(XValid5, yValid5))
testError5 = model5.evaluate(XTest5, yTest5)
print('Test Error on SimpleRNN regression:', testError5)
#if input('continue to deep RNN method with multiple prediction? Y/N')=='n':
# import sys
# sys.exit()
def build_model():
frames = None
s_size = None
# prepare shared layers
dummy_frame_inputs = Input(shape=((9, s_size, s_size, 3)))
shared_layer_h = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
shared_layer_v = Network(dummy_frame_inputs,
conv3D_branch(dummy_frame_inputs))
# build model
inputs_h = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_h')
processed_h = TimeDistributed(shared_layer_h,
name='shared_3Dconv_branch_h')(inputs_h)
inputs_v = Input(shape=((frames, 9, s_size, s_size, 3)), name='inputs_v')
processed_v = TimeDistributed(shared_layer_v,
name='shared_3Dconv_branch_v')(inputs_v)
x = Concatenate()([processed_h, processed_v])
for n_filters in [64, 32, 32, 16]:
x = TimeDistributed(
Conv2D(n_filters,
kernel_size=3,
padding='same',
activation='relu',
kernel_initializer='glorot_uniform'))(x)
x = ConvRNN2D(STConvLSTM2DCell(8,
kernel_size=3,
padding='same',
activation='tanh',
recurrent_activation='hard_sigmoid',
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal'),
return_sequences=True,
name='STConvLSTM2D')(x)
x = TimeDistributed(Lambda(lambda x: K.squeeze(x, axis=-1)),
name='squeeze')(x)
return Model(inputs=[inputs_h, inputs_v], outputs=x)
# ________________________________________________________________________________________________________________
# Layer (type) Output Shape Param # Connected to
# ================================================================================================================
# inputs_h (InputLayer) [(None, None, 9, None, None, 3)] 0
# ________________________________________________________________________________________________________________
# inputs_v (InputLayer) [(None, None, 9, None, None, 3)] 0
# ________________________________________________________________________________________________________________
# shared_3Dconv_branch_h (TimeDistributed) (None, None, None, None, 64) 279296 inputs_h[0][0]
# ________________________________________________________________________________________________________________
# shared_3Dconv_branch_v (TimeDistributed) (None, None, None, None, 64) 279296 inputs_v[0][0]
# ________________________________________________________________________________________________________________
# concatenate (Concatenate) (None, None, None, None, 128) 0 shared_3Dconv_branch_h[0][0]
# shared_3Dconv_branch_v[0][0]
# ________________________________________________________________________________________________________________
# time_distributed (TimeDistributed) (None, None, None, None, 64) 73792 concatenate[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_1 (TimeDistributed) (None, None, None, None, 32) 18464 time_distributed[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_2 (TimeDistributed) (None, None, None, None, 32) 9248 time_distributed_1[0][0]
# ________________________________________________________________________________________________________________
# time_distributed_3 (TimeDistributed) (None, None, None, None, 16) 4624 time_distributed_2[0][0]
# ________________________________________________________________________________________________________________
# STConvLSTM2D (ConvRNN2D) (None, None, None, None, 1) 40009 time_distributed_3[0][0]
# ________________________________________________________________________________________________________________
# squeeze (TimeDistributed) (None, None, None, None) 0 conv_rn_n2d[0][0]
# ================================================================================================================
# Total params: 704,729
# Trainable params: 704,633
# Non-trainable params: 96
# ________________________________________________________________________________________________________________
print(X_train.shape)
positive_counts = dict()
class_weights = ExtraSensoryHelperFunctions.calculating_class_weights(Y_train)
# reshape for timedistributed
n_steps = 8
n_length = int(SEQ_LEN / n_steps)
X_train = X_train.reshape(X_train.shape[0], n_steps, n_length, n_features)
X_test = X_test.reshape(X_test.shape[0], n_steps, n_length, n_features)
print('X_train', X_train.shape)
# build the model
# model 1
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=25, kernel_size=5, activation='relu'),
input_shape=(None, n_length, n_features)))
model.add(TimeDistributed(Conv1D(filters=25, kernel_size=5,
activation='relu')))
model.add(TimeDistributed(Conv1D(filters=25, kernel_size=5,
activation='relu')))
model.add(TimeDistributed(Conv1D(filters=25, kernel_size=5,
activation='relu')))
model.add(TimeDistributed(Conv1D(filters=25, kernel_size=5,
activation='relu')))
model.add(TimeDistributed(Dropout(0.2)))
model.add(TimeDistributed(MaxPooling1D()))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(200))
model.add(Dropout(0.2))
model.add(Dense(200, activation='relu'))
data = pd.read_feather('./feature_stage_data.ftr')
X = data[data.columns[3:]]
X = X.values
X = preprocessing.normalize(X)
X = X.reshape((X.shape[0], X.shape[1], 1))
y = data['stage']
y = pd.get_dummies(y)
y = y.values
y = preprocessing.normalize(y)
x_start = Input(shape=(21,1))
x = x_start
x = GRU(21, activation='tanh', return_sequences=True)(x)
x = TimeDistributed(Dense(5))(x)
x = Flatten()(x)
x = Dense(50,activation = 'sigmoid')(x)
x = Dense(5,activation = 'sigmoid')(x)
mod = Model(inputs=x_start, outputs=x)
mod.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy'])
mod.summary()
mod.train_on_batch(X, y)
ye = educate_dummie(y)
yp = educate_dummie(mod.predict(X))
accuracy_score(ye, yp)
# LSTM
def tennis(n_model=Sequential()):
model = n_model
###CONV LAYER
model.add(
TimeDistributed(Conv2D(3, 5, 1, activation='relu', padding='same'),
input_shape=(frames, img_width, img_height, channels)))
print(model.output_shape)
model.add(Dropout(dropout))
###CONV 2
model.add(TimeDistributed(MaxPool2D((2, 2))))
model.add(BatchNormalization())
model.add(TimeDistributed(ZeroPadding2D(1)))
model.add(
TimeDistributed(Conv2D(5, 5, 1, activation='relu', padding='same')))
print(model.output_shape)
###CONV 3
model.add(Dropout(dropout))
model.add(BatchNormalization())
model.add(
TimeDistributed(Conv2D(7, 5, 1, activation='relu', padding='same')))
##CONV 4
model.add(TimeDistributed(MaxPool2D((2, 2))))
model.add(TimeDistributed(ZeroPadding2D(1)))
model.add(
TimeDistributed(Conv2D(13, 5, 1, activation='relu', padding='same')))
###CONV 5
model.add(BatchNormalization())
model.add(Dropout(dropout))
print(model.output_shape)
model.add(
TimeDistributed(Conv2D(17, 5, 1, activation='relu', padding='same')))
####CONV 6
model.add(TimeDistributed(MaxPool2D((2, 2))))
model.add(TimeDistributed(ZeroPadding2D(1)))
model.add(
TimeDistributed(Conv2D(20, 5, 1, activation='relu', padding='same')))
###CONV 7
model.add(Dropout(dropout))
model.add(TimeDistributed(MaxPool2D((2, 2))))
model.add(
TimeDistributed(Conv2D(23, 3, 1, activation='relu', padding='same')))
###conv 8
model.add(
TimeDistributed(Conv2D(27, 3, 1, activation='relu', padding='same')))
####CONV 9
model.add(Dropout(dropout))
model.add(BatchNormalization())
model.add(TimeDistributed(MaxPool2D((2, 2))))
model.add(
TimeDistributed(Conv2D(32, 3, 1, activation='relu', padding='same')))
###conv 10
model.add(
TimeDistributed(Conv2D(35, 3, 1, activation='relu', padding='same')))
####FLATTEN LAYER
model.add(TimeDistributed(Flatten()))
print(model.output_shape)
model.add(Dropout(dropout))
####LSTM LAYER
model.add(LSTM(10, return_sequences=False))
###DENSE LAYER1
model.add(Dropout(dropout))
print(model.output_shape)
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(500, activation='relu'))
####DENSE LAYER2
model.add(BatchNormalization())
model.add(Dropout(dropout))
model.add(Dense(250, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(dropout))
#####DENSE LAYER3
model.add(Dense(150, activation='relu'))
model.add(BatchNormalization())
model.add(Dropout(dropout))
####OUTPUT LAYER
model.add(Dense(2, activation='softmax'))
return model.summary(), model
def seq2seq_model(vocabulary_size, config_params, output_size,
pos_vocab_size, lex_vocab_size, tokenizer=None,
visualize=False, plot=False):
hidden_size = int(config_params['hidden_size'])
batch_size = int(config_params['batch_size'])
embedding_size = int(config_params['embedding_size'])
input_type = 'string' if tokenizer is not None else None
in_sentences = Input(shape=(None,), dtype=input_type,
batch_size=batch_size, name='Input')
if tokenizer is not None:
embeddings = ElmoEmbeddingLayer()(in_sentences)
embedding_size = 1024
else:
embeddings = Embedding(input_dim=vocabulary_size,
output_dim=embedding_size,
mask_zero=True,
name="Embeddings")(in_sentences)
bilstm, forward_h, _, backward_h, _ = Bidirectional(LSTM(hidden_size, return_sequences=True,
return_state=True, dropout=0.2, recurrent_dropout=0.2,
input_shape=(None, None, embedding_size)),
merge_mode='sum',
name='Encoder_BiLSTM')(embeddings)
state_h = Concatenate()([forward_h, backward_h])
encoder_attention = SeqSelfAttention(attention_activation='sigmoid',
name='Attention')([bilstm, state_h])
concat = Concatenate()([encoder_attention, bilstm])
decoder_fwd_lstm, _, _ = LSTM(hidden_size, dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size),
name='Decoder_FWD_LSTM')(concat)
decoder_bck_lstm, _, _ = LSTM(hidden_size,
dropout=0.2,
recurrent_dropout=0.2,
return_sequences=True,
input_shape=(None, None, embedding_size),
go_backwards=True,
name='Decoder_BWD_LSTM')(decoder_fwd_lstm)
decoder_bilstm = Concatenate()([decoder_fwd_lstm, decoder_bck_lstm])
logits = TimeDistributed(
Dense(output_size), name='WSD_logits')(decoder_bilstm)
in_mask = Input(shape=(None, output_size),
batch_size=batch_size, name='Candidate_Synsets_Mask')
logits_mask = Add(name="Masked logits")([logits, in_mask])
pos_logits = TimeDistributed(Dense(pos_vocab_size),
name='POS_logits')(decoder_bilstm)
lex_logits = TimeDistributed(Dense(lex_vocab_size),
name='LEX_logits')(decoder_bilstm)
wsd_output = Softmax(name="WSD_output")(logits_mask)
pos_output = Softmax(name="POS_output")(pos_logits)
lex_output = Softmax(name="LEX_output")(lex_logits)
model = Model(inputs=[in_sentences, in_mask],
outputs=[wsd_output, pos_output, lex_output],
name='Seq2Seq_MultiTask')
model.compile(loss="sparse_categorical_crossentropy",
optimizer=Adadelta(), metrics=['acc'])
visualize_plot_mdl(visualize, plot, model)
return model
encoder = Bidirectional(LSTM(state_dim, return_state=True),
merge_mode='concat')(encoder_embeddings)
encoder_outputs, forward_h, forward_c, backward_h, backward_c = encoder
state_h = Concatenate()([forward_h, backward_h])
state_c = Concatenate()([forward_c, backward_c])
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(max_ans_len, ))
decoder_embeddings = embedding_layer(decoder_inputs)
decoder_lstm = LSTM(state_dim * 2, return_sequences=True, return_state=True)
decoder_outputs, _, _ = decoder_lstm(decoder_embeddings,
initial_state=encoder_states)
outputs = TimeDistributed(Dense(vocab_len,
activation='softmax'))(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], outputs)
if load:
loaded_model = load_model(path)
model.set_weights(loaded_model.get_weights())
del loaded_model
model.summary()
opt = Adam(lr=0.0000001)
model.compile(optimizer=opt, loss='sparse_categorical_crossentropy')
model.fit([encoder_input_data, decoder_input_data],
decoder_target_data,
batch_size=batch_size,
#callbacks
save_name = '../checkpoints/joint_model/joint_model5.{epoch:02d}-{val_loss:.2f}.hdf5'
best_model = ModelCheckpoint(save_name,
monitor='val_loss',
save_best_only=True,
mode='min') #save the best model
early_stopping_monitor = EarlyStopping(
patience=10) #stop training when the model is not improving
callbacks_list = [early_stopping_monitor, best_model]
#model definition
speech_input = Input(shape=(time_dim, features_dim))
gru = Bidirectional(GRU(lstm1_depth, return_sequences=True))(speech_input)
norm = BatchNormalization()(gru)
norm = TimeDistributed(Dropout(drop_prob))(norm)
reshape = Reshape((SEQ_LENGTH, norm.shape[-1] * frames_per_annotation))(norm)
speech_features = TimeDistributed(Dense(feature_vector_size,
activation='relu'))(reshape)
## conv3d network for video model
img_x = 48
img_y = 48
ch_n = 1
video_input = Input(shape=(SEQ_LENGTH, img_x, img_y, ch_n), name='video_input')
layer = TimeDistributed(
Conv2D(32,
kernel_size=(3, 3),
sequences=y,
padding="post",
value=tag2idx["O"])
x_train, x_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=1)
input_word = Input(shape=(max_len, ))
model = Embedding(input_dim=num_words, output_dim=50,
input_length=max_len)(input_word)
model = SpatialDropout1D(0.1)(model)
model = Bidirectional(
GRU(units=100, return_sequences=True, recurrent_dropout=0.1))(model)
model = TimeDistributed(Dense(num_tags))(model)
out = Activation("softmax", dtype="float32", name="predictions")(model)
model = Model(input_word, out)
model.summary()
model.compile(optimizer="adam",
loss="sparse_categorical_crossentropy",
metrics=["accuracy"])
early_stopping = EarlyStopping(
monitor="val_accuracy",
min_delta=0,
patience=1,
verbose=0,
mode="max",
baseline=None,
def get_test_model_recurrent():
"""Returns a minimalistic test model for recurrent layers."""
input_shapes = [
(17, 4),
(1, 10),
(20, 40),
(6, 7, 10, 3)
]
outputs = []
inputs = [Input(shape=s) for s in input_shapes]
inp = PReLU()(inputs[0])
lstm = Bidirectional(LSTM(units=4,
return_sequences=True,
bias_initializer='random_uniform', # default is zero use random to test computation
activation='tanh',
recurrent_activation='relu'), merge_mode='concat')(inp)
lstm2 = Bidirectional(LSTM(units=6,
return_sequences=True,
bias_initializer='random_uniform',
activation='elu',
recurrent_activation='hard_sigmoid'), merge_mode='sum')(lstm)
lstm3 = LSTM(units=10,
return_sequences=False,
bias_initializer='random_uniform',
activation='selu',
recurrent_activation='sigmoid')(lstm2)
outputs.append(lstm3)
conv1 = Conv1D(2, 1, activation='sigmoid')(inputs[1])
lstm4 = LSTM(units=15,
return_sequences=False,
bias_initializer='random_uniform',
activation='tanh',
recurrent_activation='elu')(conv1)
dense = (Dense(23, activation='sigmoid'))(lstm4)
outputs.append(dense)
time_dist_1 = TimeDistributed(Conv2D(2, (3, 3), use_bias=True))(inputs[3])
flatten_1 = TimeDistributed(Flatten())(time_dist_1)
outputs.append(Bidirectional(LSTM(units=6,
return_sequences=True,
bias_initializer='random_uniform',
activation='tanh',
recurrent_activation='sigmoid'), merge_mode='ave')(flatten_1))
outputs.append(TimeDistributed(MaxPooling2D(2, 2))(inputs[3]))
outputs.append(TimeDistributed(AveragePooling2D(2, 2))(inputs[3]))
model = Model(inputs=inputs, outputs=outputs, name='test_model_recurrent')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = 2
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, epochs=10)
return model
def train_model(base_model: keras.Model,
is_causal: bool,
tasks_meta_data: List[TaskMetadata],
pretrain_generator,
finetune_generator,
pretrain_epochs: int = 1,
pretrain_optimizer='adam',
pretrain_steps: int = 1000000,
pretrain_callbacks=None,
finetune_epochs: int = 1,
finetune_optimizer='adam',
finetune_steps: int = 10000,
finetune_callbacks=None,
verbose: int = 0):
token_input = base_model.inputs[0]
segment_input = base_model.inputs[1]
position_input = base_model.inputs[2]
uses_attn_mask = len(base_model.inputs) == 4
max_len = K.int_shape(base_model.inputs[0])[1]
if uses_attn_mask:
attention_mask_input = base_model.inputs[3]
all_logits = []
all_tasks = {task.name: task for task in tasks_meta_data}
task_nodes = {}
sent_level_mask_inputs = []
assert len(all_tasks) == len(tasks_meta_data)
for task in all_tasks.values():
task_loss_weight = Input(batch_shape=(None, 1),
dtype='float32',
name=task.name + '_loss_weight')
if task.is_token_level:
if task.name == 'lm':
decoder = Lambda(lambda x: K.dot(
x,
K.transpose(
base_model.get_layer('TokenEmbedding').weights[0])),
name='lm_logits')
else:
decoder = Dense(units=task.num_classes,
name=task.name + '_logits')
logits = TimeDistributed(decoder,
name=task.name +
'_logits_time_distributed')(Dropout(
task.dropout)(base_model.outputs[0]))
task_target = Input(batch_shape=(
None,
max_len,
),
dtype='int32',
name=task.name + '_target_input')
task_mask = Input(batch_shape=(None, max_len),
dtype='int8',
name=task.name + '_mask_input')
task_loss = Lambda(
lambda x: x[0] * masked_classification_loss(x[1], x[2], x[3]),
name=task.name + '_loss')(
[task_loss_weight, task_target, logits, task_mask])
else:
task_mask = Input(batch_shape=(None, 1),
dtype='int32',
name=task.name + '_mask_input')
decoder_input = sparse_gather(base_model.outputs[0], task_mask,
task.name)
logits = Dense(units=task.num_classes, name=task.name + '_logits')(
Dropout(task.dropout)(decoder_input))
task_target = Input(batch_shape=(None, 1),
dtype='int32',
name=task.name + '_target_input')
task_loss = Lambda(
lambda x: x[0] * classification_loss(x[1], x[2]),
name=task.name + '_loss')(
[task_loss_weight, task_target, logits])
sent_level_mask_inputs.append(task_mask)
task_nodes[task.name] = {
'target': task_target,
'mask': task_mask,
'loss_weight': task_loss_weight,
'loss': task_loss,
}
all_logits.append(logits)
def get_generator(sentence_generator: Generator[SentenceBatch, None, None],
is_pretrain: bool):
for i, batch in enumerate(sentence_generator):
batch_size, seq_len = batch.tokens.shape
x = [
batch.tokens, batch.segments,
generate_pos_ids(batch_size, max_len)
]
y = []
if uses_attn_mask:
x.append(create_attention_mask(batch.padding_mask, is_causal))
for task_name in task_nodes.keys():
if is_pretrain:
cond = all_tasks[
task_name].weight_scheduler.active_in_pretrain
else:
cond = all_tasks[
task_name].weight_scheduler.active_in_finetune
if cond:
if task_name in batch.sentence_classification:
task_data_batch = batch.sentence_classification[
task_name]
else:
task_data_batch = batch.token_classification[task_name]
x.append(task_data_batch.target)
if all_tasks[task_name].is_token_level:
x.append(task_data_batch.target_mask)
else:
x.append(
(task_data_batch.target_mask +
np.arange(batch_size) * seq_len).astype(np.int32))
x.append(
np.repeat(
np.array([
all_tasks[task_name].weight_scheduler.get(
is_pretrain, i)
]), batch_size, 0))
y.append(np.repeat(np.array([0.0]), batch_size, 0))
yield x, y
def train_step(is_pretrain: bool):
_inputs = [token_input, segment_input, position_input]
_outputs = []
if uses_attn_mask:
_inputs.append(attention_mask_input)
for task_name in task_nodes.keys():
if is_pretrain:
cond = all_tasks[task_name].weight_scheduler.active_in_pretrain
else:
cond = all_tasks[task_name].weight_scheduler.active_in_finetune
if cond:
_inputs.append(task_nodes[task_name]['target'])
_inputs.append(task_nodes[task_name]['mask'])
_inputs.append(task_nodes[task_name]['loss_weight'])
_outputs.append(task_nodes[task_name]['loss'])
_generator = get_generator(
pretrain_generator if is_pretrain else finetune_generator,
is_pretrain)
_model = keras.Model(inputs=_inputs, outputs=_outputs)
_model.compile(
pretrain_optimizer if is_pretrain else finetune_optimizer,
loss=pass_through_loss)
_model.fit_generator(
_generator,
steps_per_epoch=pretrain_steps if is_pretrain else finetune_steps,
verbose=verbose,
callbacks=pretrain_callbacks
if is_pretrain else finetune_callbacks,
shuffle=False,
epochs=pretrain_epochs if is_pretrain else finetune_epochs)
if pretrain_generator is not None:
train_step(True)
if finetune_generator is not None:
train_step(False)
return keras.Model(inputs=base_model.inputs + sent_level_mask_inputs,
outputs=all_logits)
encoderX = Input(batch_shape=(None, MAX_SEQUENCE_LEN)) encEMB = wordEmbedding(encoderX) encLSTM1 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) encLSTM2 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) ey1, eh1, ec1 = encLSTM1(encEMB) # LSTM 1층 ey2, eh2, ec2 = encLSTM2(ey1) # LSTM 2층 # Decoder decoderX = Input(batch_shape=(None, 1)) decEMB = wordEmbedding(decoderX) decLSTM1 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) decLSTM2 = LSTM(LSTM_HIDDEN, return_sequences=True, return_state=True) dy1, _, _ = decLSTM1(decEMB, initial_state=[eh1, ec1]) dy2, _, _ = decLSTM2(dy1, initial_state=[eh2, ec2]) att_dy2 = Attention(ey2, dy2) decOutput = TimeDistributed(Dense(VOCAB_SIZE, activation='softmax')) outputY = decOutput(att_dy2) # Model model = Model([encoderX, decoderX], outputY) model.load_weights(MODEL_PATH) # Chatting용 model model_enc = Model(encoderX, [eh1, ec1, eh2, ec2, ey2]) ih1 = Input(batch_shape=(None, LSTM_HIDDEN)) ic1 = Input(batch_shape=(None, LSTM_HIDDEN)) ih2 = Input(batch_shape=(None, LSTM_HIDDEN)) ic2 = Input(batch_shape=(None, LSTM_HIDDEN)) ey = Input(batch_shape=(None, MAX_SEQUENCE_LEN, LSTM_HIDDEN))
y_data = np.cos(x_axis).reshape([BATCH_SIZE, SLIP_TIMES, INPUT_SIZE])
BATCH_START += SLIP_TIMES * INPUT_SIZE
return x_data, y_data, x_axis
model = Sequential()
model.add(
LSTM(units=CELL_SIZE,
input_shape=[SLIP_TIMES, INPUT_SIZE],
batch_size=BATCH_SIZE,
return_sequences=True,
stateful=True))
# final_hidden_state if return_sequences == False top_outputs(hidden_states_each_slip)
# in this case, it can return (SLIP_TIMES, CELL_SIZE) because of 'return_sequences = True'
# True: the final state of batch1 is feed into the initial state of batch2
model.add(TimeDistributed(Dense(units=OUTPUT_SIZE)))
'''
if no TimeDistributed():
(SLIP_TIMES, CELL_SIZE) * (CELL_SIZE, OUTPUT_SIZE)
else:
for each_batch in SLIP_TIMES:
(1, CELL_SIZE) * (CELL_SIZE, OUTPUT_SIZE)
--> (SLIP_TIMES, OUTPUT_SIZE)
'''
adam = Adam(LR)
model.compile(optimizer=adam, loss='mse')
print(model.summary())
plt.ion()
plt.show()
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.backend import variable
import numpy as np
import pickle
from scipy.stats import zscore
import datetime
import pytz
np.random.seed(seed=11)
with open('series_34697_1000.pkl', 'rb') as f:
segments = pickle.load(f)
segments = zscore(segments) # standardize
deep_model = Sequential(name="LSTM-autoencoder")
deep_model.add(CuDNNGRU(100, input_shape=(1000, 1), return_sequences=True))
deep_model.add(CuDNNGRU(50, return_sequences=True))
deep_model.add(CuDNNGRU(25, return_sequences=False))
deep_model.add(Dense(25, activation=None))
deep_model.add(RepeatVector(1000))
deep_model.add(CuDNNGRU(25, return_sequences=True))
deep_model.add(CuDNNGRU(50, return_sequences=True))
deep_model.add(CuDNNGRU(100, return_sequences=True))
deep_model.add(TimeDistributed(Dense(1)))
deep_model.load_weights('lstm_autoencoder_2019-12-16_11-56-02.h5')
reco = deep_model(Input(np.reshape(segments[3], (1, 1000, 1))))
np.savetxt("reco.txt", reco, delimiter=",")
batch_size=384)
y_in = Input(shape=(512, 1))
y_err = Input(shape=(512, 1))
# h_enc = Conv1D(32, 2, activation='relu')(y_in)
# h_enc = Conv1D(32, 8, activation='relu')(h_enc)
# h_enc = CuDNNGRU(512, return_sequences=True)(y_in)
# h_enc = CuDNNGRU(256, return_sequences=True)(y_in)
h_enc = CuDNNLSTM(1024, return_sequences=False)(y_in)
# h_enc = Dense(256)(h_enc)
h_enc = Dense(16, activation=None, name='bottleneck')(h_enc)
# h_enc = BatchNormalization()(h_enc)
h_dec = RepeatVector(512)(h_enc)
h_dec = CuDNNLSTM(1024, return_sequences=True)(h_dec)
# h_dec = CuDNNGRU(256, return_sequences=True)(h_dec)
h_dec = TimeDistributed(Dense(1))(h_dec)
model = Model(inputs=[y_in, y_err], outputs=h_dec)
model.compile(optimizer=Adam(clipvalue=0.5), loss=chi2(y_err))
# model.load_weights("../../model_weights/model_2020-03-11_20-34-12.h5")
training_time_stamp = datetime.datetime.now(
tz=pytz.timezone('Europe/London')).strftime("%Y-%m-%d_%H-%M-%S")
CB = EarlyStopping(monitor='val_loss',
min_delta=5e-5,
patience=100,
verbose=1,
mode='auto')
MC = ModelCheckpoint(
'../../model_weights/model_{}.h5'.format(training_time_stamp),
def generate_model(self):
"""
Model for RNN with Encoder Decoder for S2S with attention
-------------
json config:
"arch": {
"neurons":32,
"k_reg": "None",
"k_regw": 0.1,
"rec_reg": "None",
"rec_regw": 0.1,
"drop": 0.3,
"nlayersE": 1,
"nlayersD": 1,
"activation": "relu",
"activation_r": "hard_sigmoid",
"CuDNN": false,
"rnn": "GRU",
"full": [64, 32],
"mode": "RNN_ED_s2s_att"
}
:return:
"""
neurons = self.config['arch']['neurons']
drop = self.config['arch']['drop']
nlayersE = self.config['arch']['nlayersE'] # >= 1
nlayersD = self.config['arch']['nlayersD'] # >= 1
activation = self.config['arch']['activation']
activation_r = self.config['arch']['activation_r']
activation_fl = self.config['arch']['activation_fl']
rec_reg = self.config['arch']['rec_reg']
rec_regw = self.config['arch']['rec_regw']
k_reg = self.config['arch']['k_reg']
k_regw = self.config['arch']['k_regw']
rnntype = self.config['arch']['rnn']
CuDNN = self.config['arch']['CuDNN']
# neuronsD = self.config['arch']['neuronsD']
full_layers = self.config['arch']['full']
# Extra added from training function
idimensions = self.config['idimensions']
odimensions = self.config['odimensions']
impl = self.runconfig.impl
if rec_reg == 'l1':
rec_regularizer = l1(rec_regw)
elif rec_reg == 'l2':
rec_regularizer = l2(rec_regw)
else:
rec_regularizer = None
if k_reg == 'l1':
k_regularizer = l1(k_regw)
elif rec_reg == 'l2':
k_regularizer = l2(k_regw)
else:
k_regularizer = None
RNN = LSTM if rnntype == 'LSTM' else GRU
# Encoder RNN - First Input
enc_input = Input(shape=(idimensions[0]))
encoder = RNN(neurons,
implementation=impl,
recurrent_dropout=drop,
activation=activation,
recurrent_activation=activation_r,
recurrent_regularizer=rec_regularizer,
return_sequences=True,
kernel_regularizer=k_regularizer)(enc_input)
for i in range(1, nlayersE):
encoder = RNN(neurons,
implementation=impl,
recurrent_dropout=drop,
activation=activation,
recurrent_activation=activation_r,
recurrent_regularizer=rec_regularizer,
return_sequences=True,
kernel_regularizer=k_regularizer)(encoder)
encoder_last = encoder[:, -1, :]
# Decoder RNN - Second input (Teacher Forcing)
dec_input = Input(shape=(None, 1))
decoder = RNN(neurons,
implementation=impl,
recurrent_dropout=drop,
activation=activation,
recurrent_activation=activation_r,
recurrent_regularizer=rec_regularizer,
return_sequences=True,
kernel_regularizer=k_regularizer)(dec_input)
for i in range(1, nlayersD):
decoder = RNN(neurons,
implementation=impl,
recurrent_dropout=drop,
activation=activation,
recurrent_activation=activation_r,
recurrent_regularizer=rec_regularizer,
return_sequences=True,
kernel_regularizer=k_regularizer)(decoder)
# Attention Layer
attention = dot([decoder, encoder], axes=[2, 2])
attention = Activation('softmax', name='attention')(attention)
context = dot([attention, encoder], axes=[2, 1])
# print('context', context)
decoder_combined_context = concatenate([context, decoder])
# print('decoder_combined_context', decoder_combined_context)
output = TimeDistributed(
Dense(full_layers[0],
activation=activation_fl))(decoder_combined_context)
for l in full_layers[1:]:
output = TimeDistributed(Dense(l,
activation=activation_fl))(output)
output = TimeDistributed(Dense(1, activation="linear"))(output)
self.model = Model(inputs=[enc_input, dec_input], outputs=output)
limit = n_timesteps / 4.0
y = np.array([0 if x < limit else 1 for x in np.cumsum(X)])
X = X.reshape(1, n_timesteps, 1)
y = y.reshape(1, n_timesteps, 1)
return X, y
n_units = 20
n_timesteps = 4
model = Sequential()
model.add(
Bidirectional(
LSTM(n_units, return_sequences=True, input_shape=(n_timesteps, 1))))
model.add(TimeDistributed(Dense(1, activation='sigmoid')))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
for spoch in range(1000):
X, y = get_sequence(n_timesteps)
model.fit(X, y, epochs=1, batch_size=1, verbose=2)
X, y = get_sequence(n_timesteps)
yhat = model.predict_classes(X, verbose=0)
for i in range(n_timesteps):
print('실젯값 : ', y[0, i], '예측값 : ', yhat[0, i])
def convolution_model(num_speakers=2):
# == Audio convolution layers ==
model = Sequential()
# # Implicit input layer
# inputs = Input(shape=(298, 257, 2))
# model.add(inputs)
# Convolution layers
conv1 = Conv2D(96, kernel_size=(1,7), padding='same', dilation_rate=(1,1), input_shape=(298, 257, 2), name="input_layer")
model.add(conv1)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv2 = Conv2D(96, kernel_size=(7,1), padding='same', dilation_rate=(1,1))
model.add(conv2)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv3 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(1,1))
model.add(conv3)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv4 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(2,1))
model.add(conv4)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv5 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(4,1))
model.add(conv5)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv6 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(8,1))
model.add(conv6)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv7 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(16,1))
model.add(conv7)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv8 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(32,1))
model.add(conv8)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv9 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(1,1))
model.add(conv9)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv10 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(2,2))
model.add(conv10)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv11 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(4,4))
model.add(conv11)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv12 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(8,8))
model.add(conv12)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv13 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(16,16))
model.add(conv13)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv14 = Conv2D(96, kernel_size=(5,5), padding='same', dilation_rate=(32,32))
model.add(conv14)
model.add(BatchNormalization())
model.add(Activation("relu"))
conv15 = Conv2D(8, kernel_size=(1,1), padding='same', dilation_rate=(1,1))
model.add(conv15)
model.add(BatchNormalization())
model.add(Activation("relu"))
# == AV fused neural network ==
# AV fusion step(s)
model.add(TimeDistributed(Flatten()))
# BLSTM
new_matrix_length = 400
model.add(Bidirectional(LSTM(new_matrix_length//2, return_sequences=True, input_shape=(298, 257*8))))
# Fully connected layers
model.add(Dense(600, activation="relu"))
model.add(Dense(600, activation="relu"))
model.add(Dense(600, activation="relu"))
# Output layer (i.e. complex masks)
# outputs = Dense(257*2*num_speakers, activation="relu")
outputs = Dense(257*2*num_speakers, activation="sigmoid") # TODO: check if this is more correct (based on the paper)
model.add(outputs)
outputs_complex_masks = Reshape((298, 257, 2, num_speakers), name="output_layer")
model.add(outputs_complex_masks)
# Print the output shapes of each model layer
for layer in model.layers:
name = layer.get_config()["name"]
if "batch_normal" in name or "activation" in name:
continue
print(layer.output_shape, "\t", name)
# Alternatively, print the default keras model summary
print(model.summary())
# Compile the model before training
# model.compile(optimizer='adam', loss='mse')
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
return model
# neural network model in keras
# see keras documentation for functions of different layers and structure of networks.
# the following two layers should not be changed.
input_layer = Input(shape=(max_seq_length, ))
embedding_layer = Embedding(vocab_size,
300,
weights=[embedding_vectors],
input_length=max_seq_length,
trainable=False)(input_layer)
# here, attention models have to be implemented in this model
# ...
# this last layer can/should be modified
output_layer = TimeDistributed(Dense(n_tags,
activation="elu"))(embedding_layer)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss="mean_squared_error",
optimizer="adam",
metrics=["categorical_accuracy"])
model.summary()
# fit model on train data
model.fit(x_train, y_train, batch_size=128, epochs=500)
## evaluation
# predict labels of test data
y_test_pred_prob = model.predict(x_test)
y_test_pred_sparse = y_test_pred_prob.argmax(axis=-1)
y_test_pred = to_categorical(np.array(y_test_pred_sparse), num_classes=n_tags)
def gated_residual_network(x,
hidden_layer_size,
output_size=None,
dropout_rate=None,
use_time_distributed=True,
additional_context=None,
return_gate=False,
activation: str = 'elu',
kernel_constraint1=None,
kernel_constraint2=None,
kernel_constraint3=None,
gating_kernel_constraint1=None,
gating_kernel_constraint2=None,
norm=True,
name='GRN'):
"""Applies the gated residual network (GRN) as defined in paper.
Args:
x: Network inputs
hidden_layer_size: Internal state size
output_size: Size of output layer
dropout_rate: Dropout rate if dropout is applied
use_time_distributed: Whether to apply network across time dimension
additional_context: Additional context vector to use if relevant
return_gate: Whether to return GLU gate for diagnostic purposes
name: name of all layers
activation: the kind of activation function to use
kernel_constraint1: kernel constranint to be applied on skip_connection layer
kernel_constraint2: kernel constranint to be applied on 1st linear layer before activation
kernel_constraint3: kernel constranint to be applied on 2nd linear layer after activation
gating_kernel_constraint1: kernel constraint for activation in gating layer
gating_kernel_constraint2: kernel constaint for gating layer
Returns:
Tuple of tensors for: (GRN output, GLU gate)
"""
# Setup skip connection
if output_size is None:
output_size = hidden_layer_size
skip = x
else:
linear = Dense(output_size,
name=f'skip_connection_{name}',
kernel_constraint=kernel_constraint1)
if use_time_distributed:
linear = TimeDistributed(linear, name=f'skip_connection_{name}')
skip = linear(x)
# Apply feedforward network
hidden = linear_layer(hidden_layer_size,
activation=None,
use_time_distributed=use_time_distributed,
kernel_constraint=kernel_constraint2,
name=f"ff_{name}")(x)
if additional_context is not None:
hidden = hidden + linear_layer(
hidden_layer_size,
activation=None,
use_time_distributed=use_time_distributed,
use_bias=False,
name=f'addition_cntxt_{name}')(additional_context)
hidden = Activation(activation, name=f'{name}_{activation}')(hidden)
hidden = linear_layer(hidden_layer_size,
activation=None,
kernel_constraint=kernel_constraint3,
use_time_distributed=use_time_distributed,
name=f'{name}_LastDense')(hidden)
gating_layer, gate = apply_gating_layer(
hidden,
output_size,
dropout_rate=dropout_rate,
use_time_distributed=use_time_distributed,
activation=None,
activation_kernel_constraint=gating_kernel_constraint1,
gating_kernel_constraint=gating_kernel_constraint2,
name=name)
if return_gate:
return add_and_norm([skip, gating_layer], name=name, norm=norm), gate
else:
return add_and_norm([skip, gating_layer], name=name, norm=norm)