def f(x, chosen, style):
time_steps = int(x.get_shape()[1])
# Shift target one note to the left.
shift_chosen = Lambda(lambda x: tf.pad(x[:, :, :-1, :], [[0, 0], [0, 0], [1, 0], [0, 0]]))(chosen)
# [batch, time, notes, 1]
shift_chosen = Reshape((time_steps, NUM_NOTES, -1))(shift_chosen)
# [batch, time, notes, features + 1]
x = Concatenate(axis=3)([x, shift_chosen])
for l in range(NOTE_AXIS_LAYERS):
# Integrate style
if l not in dense_layer_cache:
dense_layer_cache[l] = Dense(int(x.get_shape()[3]))
style_proj = dense_layer_cache[l](style)
style_proj = TimeDistributed(RepeatVector(NUM_NOTES))(style_proj)
style_proj = Activation('tanh')(style_proj)
style_proj = Dropout(dropout)(style_proj)
x = Add()([x, style_proj])
if l not in lstm_layer_cache:
lstm_layer_cache[l] = LSTM(NOTE_AXIS_UNITS, return_sequences=True)
x = TimeDistributed(lstm_layer_cache[l])(x)
x = Dropout(dropout)(x)
return Concatenate()([note_dense(x), volume_dense(x)])
def _collect_inputs(ae_input_shapes, ae_input_names,
conditioning_input_shapes, conditioning_input_names,):
if not isinstance(ae_input_shapes, list):
ae_input_shapes = [ae_input_shapes]
if ae_input_names is None:
ae_input_names = ['input_{}'.format(ii) for ii in range(len(ae_input_names))]
ae_inputs = []
for ii, input_shape in enumerate(ae_input_shapes):
ae_inputs.append(Input(input_shape, name='input_{}'.format(ae_input_names[ii])))
ae_stack = Concatenate(name='concat_inputs', axis=-1)(ae_inputs)
ae_stack_shape = ae_stack.get_shape().as_list()[1:]
# collect conditioning inputs, and concatentate them into a stack
if not isinstance(conditioning_input_shapes, list):
conditioning_input_shapes = [conditioning_input_shapes]
if conditioning_input_names is None:
conditioning_input_names = ['cond_input_{}'.format(ii) for ii in range(len(conditioning_input_shapes))]
conditioning_inputs = []
for ii, input_shape in enumerate(conditioning_input_shapes):
conditioning_inputs.append(Input(input_shape, name=conditioning_input_names[ii]))
cond_stack = Concatenate(name='concat_cond_inputs', axis=-1)(conditioning_inputs)
return ae_inputs, ae_stack, conditioning_inputs, cond_stack
def color_net(num_classes):
# placeholder for input image
input_image = Input(shape=(224, 224, 3))
# ============================================= TOP BRANCH ===================================================
# first top convolution layer
top_conv1 = Convolution2D(filters=48,
kernel_size=(11, 11),
strides=(4, 4),
input_shape=(224, 224, 3),
activation='relu')(input_image)
top_conv1 = BatchNormalization()(top_conv1)
top_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_conv1)
# second top convolution layer
# split feature map by half
top_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(top_conv1)
top_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(top_conv1)
top_top_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv2)
top_top_conv2 = BatchNormalization()(top_top_conv2)
top_top_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_top_conv2)
top_bot_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv2)
top_bot_conv2 = BatchNormalization()(top_bot_conv2)
top_bot_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_bot_conv2)
# third top convolution layer
# concat 2 feature map
top_conv3 = Concatenate()([top_top_conv2, top_bot_conv2])
top_conv3 = Convolution2D(filters=192,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_conv3)
# fourth top convolution layer
# split feature map by half
top_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(top_conv3)
top_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(top_conv3)
top_top_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv4)
top_bot_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv4)
# fifth top convolution layer
top_top_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_top_conv4)
top_top_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_top_conv5)
top_bot_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(top_bot_conv4)
top_bot_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(top_bot_conv5)
# ============================================= TOP BOTTOM ===================================================
# first bottom convolution layer
bottom_conv1 = Convolution2D(filters=48,
kernel_size=(11, 11),
strides=(4, 4),
input_shape=(227, 227, 3),
activation='relu')(input_image)
bottom_conv1 = BatchNormalization()(bottom_conv1)
bottom_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_conv1)
# second bottom convolution layer
# split feature map by half
bottom_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(bottom_conv1)
bottom_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(bottom_conv1)
bottom_top_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv2)
bottom_top_conv2 = BatchNormalization()(bottom_top_conv2)
bottom_top_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_top_conv2)
bottom_bot_conv2 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv2)
bottom_bot_conv2 = BatchNormalization()(bottom_bot_conv2)
bottom_bot_conv2 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_bot_conv2)
# third bottom convolution layer
# concat 2 feature map
bottom_conv3 = Concatenate()([bottom_top_conv2, bottom_bot_conv2])
bottom_conv3 = Convolution2D(filters=192,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_conv3)
# fourth bottom convolution layer
# split feature map by half
bottom_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(bottom_conv3)
bottom_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(bottom_conv3)
bottom_top_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv4)
bottom_bot_conv4 = Convolution2D(filters=96,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv4)
# fifth bottom convolution layer
bottom_top_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_top_conv4)
bottom_top_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_top_conv5)
bottom_bot_conv5 = Convolution2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
activation='relu',
padding='same')(bottom_bot_conv4)
bottom_bot_conv5 = MaxPooling2D(pool_size=(3, 3),
strides=(2, 2))(bottom_bot_conv5)
# ======================================== CONCATENATE TOP AND BOTTOM BRANCH =================================
conv_output = Concatenate()(
[top_top_conv5, top_bot_conv5, bottom_top_conv5, bottom_bot_conv5])
# Flatten
flatten = Flatten()(conv_output)
# Fully-connected layer
FC_1 = Dense(units=4096, activation='relu')(flatten)
FC_1 = Dropout(0.6)(FC_1)
FC_2 = Dense(units=4096, activation='relu')(FC_1)
FC_2 = Dropout(0.6)(FC_2)
output = Dense(units=num_classes, activation='softmax')(FC_2)
model = Model(inputs=input_image, outputs=output)
# Note: lr => learning rate
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
# sgd = SGD(lr=0.01, momentum=0.9, decay=0.0005, nesterov=True)
model.compile(optimizer=sgd,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
kernel_initializer=RandomNormal(mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05)))
baseNetwork.add(MaxPooling1D(pool_size=3, strides=3))
baseNetwork.add(Conv1D(256, 7, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05)))
baseNetwork.add(MaxPooling1D(pool_size=3, strides=3))
return baseNetwork
baseNetwork = createSplitBaseNetworkSmall(inputLength, inputDim)
# because we re-use the same instance `base_network`,
# the weights of the network will be shared across the two branches
processedA = baseNetwork(oheInputA)
processedB = baseNetwork(oheInputB)
# Concatenate
conc = Concatenate()([processedA, processedB])
x = Conv1D(256, 3, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(conc)
x = Conv1D(256, 3, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(x)
x = Conv1D(256, 3, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(x)
x = Conv1D(256, 3, strides=1, padding='valid', activation='relu', kernel_initializer=RandomNormal(
mean=0.0, stddev=0.05), bias_initializer=RandomNormal(mean=0.0, stddev=0.05))(x)
x = MaxPooling1D(pool_size=3, strides=3)(x)
x = Flatten()(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
x = Dense(1024, activation='relu')(x)
x = Dropout(0.5)(x)
def build(self):
n_steps, n_length = 4, 32
model = Sequential()
model2 = Sequential()
for i in range (2):
if i == 0:
self.train_X = self.train_X.reshape(
(self.train_X.shape[0], n_steps, n_length, self.n_features)) # self.n_features = 6
self.test_X = self.test_X.reshape(
(self.test_X.shape[0], n_steps, n_length, self.n_features))
# model = Sequential()
model.add(TimeDistributed(Conv1D(filters=64, kernel_size=12,
activation='relu'), input_shape=(None, n_length, self.n_features)))
model.add(TimeDistributed(
Conv1D(filters=16, kernel_size=3, activation='relu')))
model.add(TimeDistributed(Dropout(0.5)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(100))
model.add(Dropout(0.5))
model.add(Dense(100, activation='relu'))
# model.add(Dense(self.n_outputs, activation='softmax'))
else:
self.train_X2 = self.train_X2.reshape(
(self.train_X2.shape[0], n_steps, n_length, self.n_features)) # self.n_features = 6
self.test_X2 = self.test_X2.reshape(
(self.test_X2.shape[0], n_steps, n_length, self.n_features))
# model2 = Sequential()
model2.add(TimeDistributed(Conv1D(filters=64, kernel_size=12,
activation='relu'), input_shape=(None, n_length, self.n_features)))
model2.add(TimeDistributed(
Conv1D(filters=16, kernel_size=3, activation='relu')))
model2.add(TimeDistributed(Dropout(0.5)))
model2.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model2.add(TimeDistributed(Flatten()))
model2.add(LSTM(100))
model2.add(Dropout(0.5))
model2.add(Dense(100, activation='relu'))
# model2.add(Dense(self.n_outputs, activation='softmax'))
# x = concatenate([model, model2])
# x = Dense(self.n_outputs, activation='softmax')(x)
# concatenate them
merged = Concatenate([model, model2])
print(merged)
# self.train_X = self.train_X.reshape(
# (self.train_X.shape[0], n_steps, n_length, self.n_features))
re_train_x = Input(shape=(None, n_length, self.n_features))
re_train_X2 = Input(shape=(None, n_length, self.n_features))
# self.train_X2 = self.train_X2.reshape(
# (self.train_X2.shape[0], n_steps, n_length, self.n_features)) # self.n_features = 6
big_model = Model(inputs=[re_train_x, re_train_X2], outputs=merged)
big_model.add(Dense(self.n_outputs, activation='softmax'))
# ada_grad = Adagrad(lr=0.1, epsilon=1e-08, decay=0.0)
big_model.compile(optimizer='adam', loss='categorical_crossentropy',
metrics=['accuracy'])
# model.add(concatenate([model, model2]))
# model.add(Dense(self.n_outputs, activation='softmax'))
# model.compile(loss='categorical_crossentropy',
# optimizer='adam', metrics=['accuracy'])
super().build(big_model)
双向LSTM 获取Char embedding
"""
char_embed = Embedding(input_dim=charsize,
output_dim=char_embed_dim,
embeddings_initializer='lecun_uniform',
input_length=maxlen_char_word,
mask_zero=False,
name='char_embedding')(char_input)
s = char_embed.shape
char_embed = Lambda(lambda x: K.reshape(x, shape=(-1, s[-2], char_embed_dim)))(
char_embed)
fwd_state = GRU(150, return_state=True)(char_embed)[-2]
bwd_state = GRU(150, return_state=True, go_backwards=True)(char_embed)[-2]
char_embed = Concatenate(axis=-1)([fwd_state, bwd_state])
char_embed = Lambda(lambda x: K.reshape(x, shape=[-1, s[1], 2 * 150]))(
char_embed)
char_embed = Dropout(0.5, name='char_embed_dropout')(char_embed)
"""
使用attention将word和character embedding结合起来
"""
W_embed = Dense(300, name='Wembed')(embed)
W_char_embed = Dense(300, name='W_charembed')(char_embed)
merged1 = merge([W_embed, W_char_embed], name='merged1', mode='sum')
tanh = Activation('tanh', name='tanh')(merged1)
W_tanh = Dense(300, name='w_tanh')(tanh)
a = Activation('sigmoid', name='sigmoid')(W_tanh)
t = Lambda(lambda x: K.ones_like(x, dtype='float32'))(a)
from keras.layers.pooling import GlobalAveragePooling2D
from keras.layers import Activation
from keras.callbacks import EarlyStopping
from keras.optimizers import Adam
from keras.datasets import cifar10
from keras.utils import np_utils
from keras.layers.normalization import BatchNormalization
input_1 = Input(shape=(32, 32, 3), name="head")
x1 = Conv2D(16, (3, 3), padding='same', name='2-1')(input_1)
x2 = Conv2D(16, (3, 3), padding='same', name='2-2')(input_1)
x3 = Conv2D(16, (3, 3), padding='same', name='2-3')(input_1)
x4 = Conv2D(10, (3, 3), padding='same', name='3-1')(x1)
x5 = Concatenate(axis=3, name="3-2")([x1, x2])
x6 = Concatenate(axis=3, name="3-3")([x2, x3])
x7 = Conv2D(10, (3, 3), padding='same', name='3-4')(x3)
out1 = GlobalAveragePooling2D(name="4-1")(x4)
out1 = Activation('softmax', name='r_hand')(out1)
x10 = Concatenate(axis=3, name="4-2")([x5, x6])
out4 = GlobalAveragePooling2D(name="4-3")(x7)
out4 = Activation('softmax', name='l_hand')(out4)
x11 = MaxPooling2D(pool_size=(2, 2), name="5-1")(x10)
out2 = Flatten(name="6-1")(x11)
out2 = Dense(10, activation='relu', name="7-1")(out2)
x_dense = Dropout(0.3)(x_dense)
output1 = Dense(5)(x_dense)
# 모델 2
input2 = Input(shape=(5,5))
y_dense = LSTM(128, activation='relu')(input2)
y_dense = Dense(256)(y_dense)
y_dense = Dense(256)(y_dense)
y_dense = Dense(256)(y_dense)
y_dense = Dense(256)(y_dense)
y_dense = Dropout(0.3)(y_dense)
output2 = Dense(5)(y_dense)
from keras.layers.merge import Concatenate # from keras.layers import Concatenate
merge1 = Concatenate()([output1, output2])
dense = Dense(32)(merge1)
doutput= Dense(32)(dense)
output_final = Dense(1)(dense)
model = Model(inputs=[input1, input2], outputs=output_final)
# 5. 모델 훈련
from keras.callbacks import EarlyStopping, TensorBoard
td_hist = TensorBoard(log_dir='./graph',
histogram_freq=0,
write_graph=True,
write_images=True)
def CNN_procedures():
# Laoding the data set - training data.
corpus = load_files(container_path='bbcsport',
description=None,
load_content=True,
encoding='utf-8',
categories=categories,
shuffle=True,
decode_error='ignore',
random_state=42)
# You can check the target names ( categories )
print(corpus.target_names)
print(len(corpus.data))
texts = []
labels = corpus.target
print('labels', labels)
texts = corpus.data
tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
words_index = tokenizer.word_index
print('Found %s unique tokens.' % len(words_index))
data = pad_sequences(sequences, maxlen=MAX_SEQUENCE_LENGTH)
labels = to_categorical(np.asarray(labels))
print("Shape of data tensor:", data.shape)
print("Shape of label tensor:", labels.shape)
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
nb_validations_samples = int(VALIDATION_SPLIT * data.shape[0])
print(nb_validations_samples)
x_train = data[:-nb_validations_samples]
y_train = labels[:-nb_validations_samples]
x_val = data[-nb_validations_samples:]
y_val = labels[-nb_validations_samples:]
print('Number of positive and negative in training and validation set')
print(y_train.sum(axis=0))
print(y_val.sum(axis=0))
GLOVE_DIR = "glove.6B"
embeddings_index = {}
glove_file = open(os.path.join(GLOVE_DIR, 'glove.6B.100d.txt'))
for line in glove_file:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
glove_file.close()
embedding_matrix = np.random.random((len(words_index) + 1, EMBEDDING_DIM))
for word, i in words_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# Words not found in embedding index will be all-zeros
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(words_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
# Complex convulotional approach
convs = []
filter_sizes = [3, 4, 5]
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
for fsz in filter_sizes:
l_conv = Conv1D(nb_filter=128, filter_length=fsz,
activation='relu')(embedded_sequences)
l_pool = MaxPooling1D(5)(l_conv)
convs.append(l_pool)
l_merge = Concatenate(axis=1)(convs)
l_cov1 = Conv1D(128, 5, activation='relu')(l_merge)
l_pool1 = MaxPooling1D(5)(l_cov1)
l_cov2 = Conv1D(128, 5, activation='relu')(l_pool1)
l_pool2 = MaxPooling1D(30)(l_cov2)
l_flat = Flatten()(l_pool2)
l_dense = Dense(128, activation='relu')(l_flat)
preds = Dense(5, activation='softmax')(l_dense)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
print("Model fitting - more complex convolutiona neural network")
plot_model(model, to_file='model_non_static.png')
model.summary()
history = model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
epochs=10,
batch_size=50)
# Plot training & valition accuracy values
plt.plot(history.history['acc']) # accuracy
plt.plot(history.history['val_acc']) # validation accuracy
plt.title('Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch') # Number of iteration
plt.legend(['accuracy', 'validation accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(history.history['loss']) # Training loss
plt.plot(history.history['val_loss']) # validation loss
plt.title('Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['loss', 'validation loss'], loc='upper left')
plt.show()
print('word_sequence_valid shape:', word_sequence_valid.shape)
print('tag_sequence_valid shape:', tag_sequence_valid.shape)
#pickle.dump(word_to_id, open("output/word_to_id.pkl", 'wb'))
#pickle.dump(char_to_id, open("output/char_to_id.pkl", 'wb'))
#pickle.dump(tag_to_id, open("output/tag_to_id.pkl", 'wb'))
print('Train...')
char_input = Input(shape=(maxCharSize * max_words,), dtype='int32', name='char_input')
char_emb = Embedding(char_vocab_size, char_embedding_dim, embeddings_initializer=my_init, input_length = max_words*maxCharSize, name='char_emb')(char_input)
char_emb = Dropout(0.5)(char_emb)
char_cnn = Conv1D(filters=30, kernel_size=3, activation='relu', padding='same')(char_emb)
char_max_pooling = MaxPooling1D(pool_size=maxCharSize)(char_cnn)
word_input = Input(shape=(max_words,), dtype='int32', name='word_input')
word_emb = Embedding(word_vocab_size+1, word_embedding_dim, weights=[embedding_matrix], name='word_emb')(word_input)
final_emb = Concatenate(axis=2, name='final_emb')([word_emb, char_max_pooling])
emb_droput = Dropout(0.5)(final_emb)
bilstm_word = Bidirectional(LSTM(200, kernel_initializer='glorot_uniform', return_sequences=True, unit_forget_bias=True))(emb_droput)
bilstm_word_d = Dropout(0.5)(bilstm_word)
crf = CRF(tag_label_size, learn_mode='marginal', sparse_target=False, use_boundary = False)
crf_output = crf(bilstm_word_d)
model = Model(inputs=[char_input, word_input], outputs=crf_output)
sgd = SGD(lr=0.015, decay=0.05, momentum=0.9, nesterov=False, clipvalue=5)
model.compile(loss=crf.loss_function, optimizer=sgd, metrics=[crf.accuracy])
filepath = "output/lstm-{epoch:02d}-{loss:.4f}.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')
callbacks_list=[checkpoint]
def __init__(self, **kwargs):
self.is_placeholder = True
super(VaeLoss, self).__init__(**kwargs)
def call(self, inputs):
x, z_m, z_l_v, d_m, d_l_v = inputs
kl_loss = -0.5 * K.sum(1. + z_l_v - K.square(z_m) - K.exp(z_l_v),
axis=1)
dec_loss = K.sum(0.5 * d_l_v + 0.5 * K.square(x - d_m) / K.exp(d_l_v),
axis=1)
vae_loss = K.mean(kl_loss + dec_loss)
self.add_loss(vae_loss, inputs=inputs)
return vae_loss
con = Concatenate(axis=-1)
y = VaeLoss()([_x, z_mean, z_log_var, dec_mean, dec_log_var])
vae = Model(_x, y)
vae.compile(optimizer='adam', loss=None)
enc = Model(_x, z_mean)
vae_ = Model(_x, con([dec_mean, dec_log_var]))
#vae.summary()
plt.clf()
epoch_splits = 5
epoch_period = epoch // epoch_splits
fig, ax = plt.subplots(epoch_splits, 2, figsize=(16, epoch_splits * 8))
if batch_size == 'Full':
batch_size = len(train_x)
def Build_Model_CNN_Text(word_index,
embeddings_index,
nclasses,
MAX_SEQUENCE_LENGTH,
EMBEDDING_DIM,
sparse_categorical,
min_hidden_layer_cnn,
max_hidden_layer_cnn,
min_nodes_cnn,
max_nodes_cnn,
random_optimizor,
dropout,
simple_model=False):
"""
def buildModel_CNN(word_index,embeddings_index,nClasses,MAX_SEQUENCE_LENGTH,EMBEDDING_DIM,Complexity=0):
word_index in word index ,
embeddings_index is embeddings index, look at data_helper.py
nClasses is number of classes,
MAX_SEQUENCE_LENGTH is maximum lenght of text sequences,
EMBEDDING_DIM is an int value for dimention of word embedding look at data_helper.py
Complexity we have two different CNN model as follows
F=0 is simple CNN with [1 5] hidden layer
Complexity=2 is more complex model of CNN with filter_length of range [1 10]
"""
model = Sequential()
if simple_model:
embedding_matrix = np.random.random(
(len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
if len(embedding_matrix[i]) != len(embedding_vector):
print(
"could not broadcast input array from shape",
str(len(embedding_matrix[i])), "into shape",
str(len(embedding_vector)), " Please make sure your"
" EMBEDDING_DIM is equal to embedding_vector file ,GloVe,"
)
exit(1)
# words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
model.add(
Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True))
values = list(range(min_nodes_cnn, max_nodes_cnn))
Layer = list(range(min_hidden_layer_cnn, max_hidden_layer_cnn))
Layer = random.choice(Layer)
for i in range(0, Layer):
Filter = random.choice(values)
model.add(Conv1D(Filter, 5, activation='relu'))
model.add(Dropout(dropout))
model.add(MaxPooling1D(5))
model.add(Flatten())
Filter = random.choice(values)
model.add(Dense(Filter, activation='relu'))
model.add(Dropout(dropout))
Filter = random.choice(values)
model.add(Dense(Filter, activation='relu'))
model.add(Dropout(dropout))
model.add(Dense(nclasses, activation='softmax'))
model_tmp = model
#model = Model(sequence_input, preds)
if sparse_categorical:
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizors(random_optimizor),
metrics=['accuracy'])
else:
model.compile(loss='categorical_crossentropy',
optimizer=optimizors(random_optimizor),
metrics=['accuracy'])
else:
embedding_matrix = np.random.random(
(len(word_index) + 1, EMBEDDING_DIM))
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
# words not found in embedding index will be all-zeros.
if len(embedding_matrix[i]) != len(embedding_vector):
print(
"could not broadcast input array from shape",
str(len(embedding_matrix[i])), "into shape",
str(len(embedding_vector)), " Please make sure your"
" EMBEDDING_DIM is equal to embedding_vector file ,GloVe,"
)
exit(1)
embedding_matrix[i] = embedding_vector
embedding_layer = Embedding(len(word_index) + 1,
EMBEDDING_DIM,
weights=[embedding_matrix],
input_length=MAX_SEQUENCE_LENGTH,
trainable=True)
# applying a more complex convolutional approach
convs = []
values_layer = list(range(min_hidden_layer_cnn, max_hidden_layer_cnn))
filter_sizes = []
layer = random.choice(values_layer)
print("Filter ", layer)
for fl in range(0, layer):
filter_sizes.append((fl + 2))
values_node = list(range(min_nodes_cnn, max_nodes_cnn))
node = random.choice(values_node)
print("Node ", node)
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
for fsz in filter_sizes:
l_conv = Conv1D(node, kernel_size=fsz,
activation='relu')(embedded_sequences)
l_pool = MaxPooling1D(5)(l_conv)
#l_pool = Dropout(0.25)(l_pool)
convs.append(l_pool)
l_merge = Concatenate(axis=1)(convs)
l_cov1 = Conv1D(node, 5, activation='relu')(l_merge)
l_cov1 = Dropout(dropout)(l_cov1)
l_pool1 = MaxPooling1D(5)(l_cov1)
l_cov2 = Conv1D(node, 5, activation='relu')(l_pool1)
l_cov2 = Dropout(dropout)(l_cov2)
l_pool2 = MaxPooling1D(30)(l_cov2)
l_flat = Flatten()(l_pool2)
l_dense = Dense(1024, activation='relu')(l_flat)
l_dense = Dropout(dropout)(l_dense)
l_dense = Dense(512, activation='relu')(l_dense)
l_dense = Dropout(dropout)(l_dense)
preds = Dense(nclasses, activation='softmax')(l_dense)
model = Model(sequence_input, preds)
model_tmp = model
if sparse_categorical:
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizors(random_optimizor),
metrics=['accuracy'])
else:
model.compile(loss='categorical_crossentropy',
optimizer=optimizors(random_optimizor),
metrics=['accuracy'])
return model, model_tmp
def get_training_model(weight_decay,
np_branch1,
np_branch2,
stages=6,
gpus=None):
img_input_shape = (None, None, 3)
vec_input_shape = (None, None, np_branch1)
heat_input_shape = (None, None, np_branch2)
inputs = []
outputs = []
img_input = Input(shape=img_input_shape)
vec_weight_input = Input(shape=vec_input_shape)
heat_weight_input = Input(shape=heat_input_shape)
inputs.append(img_input)
if np_branch1 > 0:
inputs.append(vec_weight_input)
if np_branch2 > 0:
inputs.append(heat_weight_input)
#img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5]
img_normalized = img_input # will be done on augmentation stage
# VGG
stage0_out = vgg_block(img_normalized, weight_decay)
# stage 1 - branch 1 (PAF)
new_x = []
if np_branch1 > 0:
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1,
weight_decay)
w1 = apply_mask(stage1_branch1_out, vec_weight_input,
heat_weight_input, np_branch1, 1, 1, np_branch1,
np_branch2)
outputs.append(w1)
new_x.append(stage1_branch1_out)
# stage 1 - branch 2 (confidence maps)
if np_branch2 > 0:
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2,
weight_decay)
w2 = apply_mask(stage1_branch2_out, vec_weight_input,
heat_weight_input, np_branch2, 1, 2, np_branch1,
np_branch2)
outputs.append(w2)
new_x.append(stage1_branch2_out)
new_x.append(stage0_out)
x = Concatenate()(new_x)
# stage sn >= 2
for sn in range(2, stages + 1):
new_x = []
# stage SN - branch 1 (PAF)
if np_branch1 > 0:
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1,
weight_decay)
w1 = apply_mask(stageT_branch1_out, vec_weight_input,
heat_weight_input, np_branch1, sn, 1, np_branch1,
np_branch2)
outputs.append(w1)
new_x.append(stageT_branch1_out)
# stage SN - branch 2 (confidence maps)
if np_branch2 > 0:
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2,
weight_decay)
w2 = apply_mask(stageT_branch2_out, vec_weight_input,
heat_weight_input, np_branch2, sn, 2, np_branch1,
np_branch2)
outputs.append(w2)
new_x.append(stageT_branch2_out)
new_x.append(stage0_out)
if sn < stages:
x = Concatenate()(new_x)
model = Model(inputs=inputs, outputs=outputs)
return model
ratings['rating'] = ratings['rating'].apply(convert_int_2)
y = np.zeros((ratings.shape[0], nClass))
y[np.arange(ratings.shape[0]), ratings['rating']] = 1
movie_input = Input(shape=(1, ))
movie_vec = Flatten()(Embedding(n_movies + 1, embedding_size)(movie_input))
movie_vec = Dropout(0.5)(movie_vec)
user_input = Input(shape=(1, ))
user_vec = Flatten()(Embedding(n_users + 1, embedding_size)(user_input))
user_vec = Dropout(0.5)(user_vec)
input_vecs = Concatenate()([movie_vec, user_vec])
nn = Dropout(0.5)(Dense(128, activation='relu')(input_vecs))
nn = BatchNormalization()(nn)
nn = Dropout(0.5)(Dense(128, activation='relu')(nn))
nn = BatchNormalization()(nn)
nn = Dense(128, activation='relu')(nn)
result = Dense(nClass, activation='softmax')(nn)
model = Model([movie_input, user_input], result)
model.compile('adam', 'categorical_crossentropy', metrics=['accuracy'])
model.load_weights('models/final_embedding_weights-03-0.50.hdf5')
def hybrid_recommandation(userId, idx, idx_movie):
tmdbId = int(indices_map_for_tmdb['id'][idx])
title = md.loc[md['id'] == tmdbId]['title']
def get_model(i):
momentum_lr = 0
lr_rate = 0
dr_img_1 = 0
dr_img_2 = 0
dr_end = 0
dense_coef = 0
decay_lr = 0
if (i == 0):
momentum_lr = 0.91
lr_rate = 0.0001
dr_img_1 = 0.1
dr_img_2 = 0.1
dr_end = 0.5
dense_coef = 64
decay_lr = 0.0
if (i == 1):
momentum_lr = 0.92
lr_rate = 0.0001
dr_img_1 = 0.1
dr_img_2 = 0.1
dr_end = 0.5
dense_coef = 64
decay_lr = 0.0
if (i == 2):
momentum_lr = 0.93
lr_rate = 0.0001
dr_img_1 = 0.1
dr_img_2 = 0.1
dr_end = 0.5
dense_coef = 64
decay_lr = 0.0
if (i == 3):
momentum_lr = 0.91
lr_rate = 0.0001
dr_img_1 = 0.2
dr_img_2 = 0.2
dr_end = 0.6
dense_coef = 64
decay_lr = 0.0005
if (i == 4):
momentum_lr = 0.99
lr_rate = 0.0001
dr_img_1 = 0.2
dr_img_2 = 0.2
dr_end = 0.5
dense_coef = 32
decay_lr = 0.0005
bn_model = momentum_lr
p_activation = "elu"
drop_img1 = dr_img_1
drop_img2 = dr_img_2
input_1 = Input(shape=(75, 75, 3), name="X_1")
input_2 = Input(shape=[1], name="angle")
img_1 = Conv2D(16, kernel_size=(3, 3), activation=p_activation)(
(BatchNormalization(momentum=bn_model))(input_1))
img_1 = Conv2D(16, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = Conv2D(16, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = Conv2D(16, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(drop_img1)(img_1)
img_1 = Conv2D(32, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = Conv2D(32, kernel_size=(3, 3), activation=p_activation)(img_1)
# img_1 = MaxPooling2D((2,2)) (img_1)
img_1 = Conv2D(32, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = Conv2D(32, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(drop_img1)(img_1)
img_1 = Conv2D(64, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = Conv2D(64, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(drop_img1)(img_1)
img_1 = Conv2D(128, kernel_size=(3, 3), activation=p_activation)(img_1)
img_1 = MaxPooling2D((2, 2))(img_1)
img_1 = Dropout(drop_img1)(img_1)
# img_1 = Conv2D(256, kernel_size = (3,3), activation=p_activation) (img_1)
# img_1 = MaxPooling2D((2,2)) (img_1)
# img_1 = Dropout(0.2)(img_1)
img_1 = GlobalMaxPooling2D()(img_1)
# img_2 = Conv2D(16, kernel_size = (3,3), activation=p_activation) ((BatchNormalization(momentum=bn_model))(input_1))
# img_2 = Conv2D(16, kernel_size = (3,3), activation=p_activation) (img_2)
# img_2 = MaxPooling2D((2,2)) (img_2)
# img_2 = Dropout(drop_img2)(img_2)
#img_2 = Conv2D(32, kernel_size = (3,3), activation=p_activation) ((BatchNormalization(momentum=bn_model))(input_1))
#img_2 = Conv2D(32, kernel_size = (3,3), activation=p_activation) (img_2)
#img_2 = MaxPooling2D((2,2)) (img_2)
#img_2 = Dropout(drop_img2)(img_2)
img_2 = Conv2D(64, kernel_size=(3, 3), activation=p_activation)(
(BatchNormalization(momentum=bn_model))(input_1))
img_2 = MaxPooling2D((2, 2))(img_2)
img_2 = Dropout(drop_img2)(img_2)
img_2 = Conv2D(128, kernel_size=(3, 3), activation=p_activation)(
(BatchNormalization(momentum=bn_model))(img_2))
img_2 = MaxPooling2D((2, 2))(img_2)
img_2 = Dropout(drop_img2)(img_2)
img_2 = GlobalMaxPooling2D()(img_2)
img_concat = (Concatenate()(
[img_1, img_2,
BatchNormalization(momentum=bn_model)(input_2)]))
dense_ayer = Dropout(dr_end)(BatchNormalization(momentum=bn_model)(Dense(
dense_coef * 4, activation=p_activation)(img_concat)))
dense_ayer = Dropout(dr_end)(BatchNormalization(momentum=bn_model)(Dense(
dense_coef, activation=p_activation)(dense_ayer)))
output = Dense(1, activation="sigmoid")(dense_ayer)
model = Model([input_1, input_2], output)
optimizer = Adam(lr=lr_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=decay_lr)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
return model
def build_model( baseline_cnn = False ):
#Based on kernel https://www.kaggle.com/devm2024/keras-model-for-beginners-0-210-on-lb-eda-r-d
image_input = Input( shape = (75, 75, 3), name = 'images' )
angle_input = Input( shape = [1], name = 'angle' )
activation = 'elu'
bn_momentum = 0.99
# Simple CNN as baseline model
if baseline_cnn:
model = Sequential()
model.add( Conv2D(16, kernel_size = (3, 3), activation = 'relu', input_shape = (75, 75, 3)) )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( MaxPooling2D(pool_size = (3, 3), strides = (2, 2)) )
model.add( Dropout(0.2) )
model.add( Conv2D(32, kernel_size = (3, 3), activation = 'relu') )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( MaxPooling2D(pool_size = (2, 2), strides = (2, 2)) )
model.add( Dropout(0.2) )
model.add( Conv2D(64, kernel_size = (3, 3), activation = 'relu') )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( MaxPooling2D(pool_size = (2, 2), strides = (2, 2)) )
model.add( Dropout(0.2) )
model.add( Conv2D(128, kernel_size = (3, 3), activation = 'relu') )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( MaxPooling2D(pool_size = (2, 2), strides = (2, 2)) )
model.add( Dropout(0.2) )
model.add( Flatten() )
model.add( Dense(256, activation = 'relu') )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( Dropout(0.3) )
model.add( Dense(128, activation = 'relu') )
model.add( BatchNormalization(momentum = bn_momentum) )
model.add( Dropout(0.3) )
model.add( Dense(1, activation = 'sigmoid') )
opt = Adam( lr = 1e-3, beta_1 = .9, beta_2 = .999, decay = 1e-3 )
model.compile( loss = 'binary_crossentropy', optimizer = opt, metrics = ['accuracy'] )
model.summary()
else:
img_1 = Conv2D( 32, kernel_size = (3, 3), activation = activation, padding = 'same' ) ((BatchNormalization(momentum=bn_momentum) ) ( image_input) )
img_1 = MaxPooling2D( (2,2)) (img_1 )
img_1 = Dropout( 0.2 )( img_1 )
img_1 = Conv2D( 64, kernel_size = (3, 3), activation = activation, padding = 'same' ) ( (BatchNormalization(momentum=bn_momentum)) (img_1) )
img_1 = MaxPooling2D( (2,2) ) ( img_1 )
img_1 = Dropout( 0.2 )( img_1 )
# Residual block
img_2 = Conv2D( 128, kernel_size = (3, 3), activation = activation, padding = 'same' ) ( (BatchNormalization(momentum=bn_momentum)) (img_1) )
img_2 = Dropout(0.2) ( img_2 )
img_2 = Conv2D( 64, kernel_size = (3, 3), activation = activation, padding = 'same' ) ( (BatchNormalization(momentum=bn_momentum)) (img_2) )
img_2 = Dropout(0.2) ( img_2 )
img_res = add( [img_1, img_2] )
# Filter resudial output
img_res = Conv2D( 128, kernel_size = (3, 3), activation = activation ) ( (BatchNormalization(momentum=bn_momentum)) (img_res) )
img_res = MaxPooling2D( (2,2) ) ( img_res )
img_res = Dropout( 0.2 )( img_res )
img_res = GlobalMaxPooling2D() ( img_res )
cnn_out = ( Concatenate()( [img_res, BatchNormalization(momentum=bn_momentum)(angle_input)]) )
dense_layer = Dropout( 0.5 ) ( BatchNormalization(momentum=bn_momentum) (Dense(256, activation = activation) (cnn_out)) )
dense_layer = Dropout( 0.5 ) ( BatchNormalization(momentum=bn_momentum) (Dense(64, activation = activation) (dense_layer)) )
output = Dense( 1, activation = 'sigmoid' ) ( dense_layer )
model = Model( [image_input, angle_input], output )
opt = Adam( lr = 1e-3, beta_1 = .9, beta_2 = .999, decay = 1e-3 )
model.compile( loss = 'binary_crossentropy', optimizer = opt, metrics = ['accuracy'] )
model.summary()
return model
def transform_encoder_model(input_shapes, input_names=None,
latent_shape=(50,),
model_name='VTE_transform_encoder',
enc_params=None,
):
'''
Generic encoder for a stack of inputs
:param input_shape:
:param latent_shape:
:param model_name:
:param enc_params:
:return:
'''
if not isinstance(input_shapes, list):
input_shapes = [input_shapes]
if input_names is None:
input_names = ['input_{}'.format(ii) for ii in range(len(input_shapes))]
inputs = []
for ii, input_shape in enumerate(input_shapes):
inputs.append(Input(input_shape, name='input_{}'.format(input_names[ii])))
inputs_stacked = Concatenate(name='concat_inputs', axis=-1)(inputs)
input_stack_shape = inputs_stacked.get_shape().as_list()[1:]
n_dims = len(input_stack_shape) - 1
x_transform_enc = basic_networks.encoder(
x=inputs_stacked,
img_shape=input_stack_shape,
conv_chans=enc_params['nf_enc'],
min_h=None, min_c=None,
n_convs_per_stage=enc_params['n_convs_per_stage'],
use_residuals=enc_params['use_residuals'],
use_maxpool=enc_params['use_maxpool'],
prefix='vte'
)
latent_size = np.prod(latent_shape)
if not enc_params['fully_conv']:
# the last layer in the basic encoder will be a convolution, so we should activate after it
x_transform_enc = LeakyReLU(0.2)(x_transform_enc)
x_transform_enc = Flatten()(x_transform_enc)
z_mean = Dense(latent_size, name='latent_mean',
kernel_initializer=keras_init.RandomNormal(mean=0., stddev=0.00001))(x_transform_enc)
z_logvar = Dense(latent_size, name='latent_logvar',
bias_initializer=keras_init.RandomNormal(mean=-2., stddev=1e-10),
kernel_initializer=keras_init.RandomNormal(mean=0., stddev=1e-10),
)(x_transform_enc)
else:
emb_shape = basic_networks.get_encoded_shape(input_stack_shape, conv_chans=enc_params['nf_enc'])
n_chans = emb_shape[-1]
if n_dims == 3:
# convolve rather than Lambda since we want to set the initialization
z_mean = Conv3D(latent_shape[-1], kernel_size=2, strides=2, padding='same',
kernel_initializer=keras_init.RandomNormal(mean=0., stddev=0.001))(x_transform_enc)
z_mean = Flatten(name='latent_mean')(z_mean)
z_logvar = Conv3D(latent_shape[-1], kernel_size=2,
strides=2, padding='same',
bias_initializer=keras_init.RandomNormal(mean=-2., stddev=1e-10),
kernel_initializer=keras_init.RandomNormal(mean=0., stddev=1e-10),
)(x_transform_enc)
z_logvar = Flatten(name='latent_logvar')(z_logvar)
else:
# TODO: also convolve to latent mean and logvar for 2D?
z_mean = Lambda(lambda x: x[:, :, :, :n_chans/2],
output_shape=emb_shape[:-1] + (n_chans/2,))(x_transform_enc)
z_mean = Flatten(name='latent_mean')(z_mean)
z_logvar = Lambda(lambda x: x[:, :, :, n_chans/2:],
output_shape=emb_shape[:-1] + (n_chans/2,))(x_transform_enc)
z_logvar = Flatten(name='latent_logvar')(z_logvar)
return Model(inputs=inputs, outputs=[z_mean, z_logvar], name=model_name)
def get_training_model_new(weight_decay,
np1=38,
np2=19,
stages=6,
keras_preprocessing=False):
"""
Keras training model. It would output the model to train.
:param weight_decay : weight_decay factor
:np1 : Number of vectormaps usually np2*2
:np2 : Number of heatmaps in the model. Usually the same as number of bodyparts.
:Stages: How many refining stages.
:Keras preprocessing: wheter to use keras given preprocessing function to pass through vgg19 or not.
If not a -0.5,0.5 normalization would fine.
"""
stages = stages
np_branch1 = np1
np_branch2 = np2
img_input_shape = (None, None, 3)
vec_input_shape = (None, None, np1)
heat_input_shape = (None, None, np2)
inputs = []
outputs = []
img_input = Input(shape=img_input_shape)
vec_weight_input = Input(shape=vec_input_shape)
heat_weight_input = Input(shape=heat_input_shape)
inputs.append(img_input)
inputs.append(vec_weight_input)
inputs.append(heat_weight_input)
if keras_preprocessing:
img_normalized = preprocess_input(img_input)
else:
img_normalized = Lambda(lambda x: x / 256 - 0.5)(
img_input) # [-0.5, 0.5]
# VGG
stage0_out = vgg_block(img_normalized, weight_decay)
# stage 1 - branch 1 (PAF)
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1, weight_decay)
w1 = apply_mask(stage1_branch1_out, vec_weight_input, heat_weight_input,
np_branch1, 1, 1, np1)
# stage 1 - branch 2 (confidence maps)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, weight_decay)
w2 = apply_mask(stage1_branch2_out, vec_weight_input, heat_weight_input,
np_branch2, 1, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out])
outputs.append(w1)
outputs.append(w2)
# stage sn >= 2
for sn in range(2, stages + 1):
# stage SN - branch 1 (PAF)
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1, weight_decay)
w1 = apply_mask(stageT_branch1_out, vec_weight_input,
heat_weight_input, np_branch1, sn, 1, np1)
# stage SN - branch 2 (confidence maps)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, weight_decay)
w2 = apply_mask(stageT_branch2_out, vec_weight_input,
heat_weight_input, np_branch2, sn, 2)
outputs.append(w1)
outputs.append(w2)
if (sn < stages):
x = Concatenate()(
[stageT_branch1_out, stageT_branch2_out, stage0_out])
model = Model(inputs=inputs, outputs=outputs)
return model
def __init__(self,
hid_dim,
batch_norm,
dropout,
recurrent_dropout,
channels,
num_classes=1,
depth=1,
input_dim=76,
model_size=4,
**kwargs):
self.hid_dim = hid_dim
self.batch_norm = batch_norm
self.dropout = dropout
self.recurrent_dropout = recurrent_dropout
self.depth = depth
self.model_size = model_size
# Input layers and masking
X = Input(shape=(None, input_dim), name='X')
inputs = [X]
mask_X = Masking()(X)
# Preprocess each channel
channels_X = []
for ch in channels:
channels_X.append(Slice(ch)(mask_X))
channel_lstm__X = [] # LSTM processed version of channels_X
for cx in channels_X:
lstm_cx = cx
for i in range(depth):
units = hid_dim // 2
lstm = LSTM(units=units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout)
lstm_cx = Bidirectional(lstm)(lstm_cx)
channel_lstm__X.append(lstm_cx)
# Concatenate processed channels
CWX = Concatenate(axis=2)(channel_lstm__X)
# LSTM for time series lstm processed data
for i in range(depth -
1): # last layer is left for manipulation of output.
units = int(model_size * hid_dim) // 2
lstm = LSTM(units=units,
activation='tanh',
return_sequences=True,
dropout=dropout,
recurrent_dropout=recurrent_dropout)
CWX = Bidirectional(lstm)(CWX)
# Output module of the network
last_layer = LSTM(units=int(model_size * hid_dim),
activation='tanh',
return_sequences=False,
dropout=dropout,
recurrent_dropout=recurrent_dropout)(CWX)
if dropout > 0:
last_layer = Dropout(dropout)(last_layer)
y = Dense(num_classes, activation='sigmoid')(last_layer)
outputs = [y]
super(channel_wise_lstms, self).__init__(inputs=inputs,
outputs=outputs)
for i in range(1, len(XtrainList)):
maxXval = np.maximum(maxXval, XtestList[i].toarray())
minXval = np.minimum(minXval, XtestList[i].toarray())
sumXval = sumXval + XtestList[i].toarray()
Y_val = np_utils.to_categorical(Y_test - 1, 1000)
tensorboard = TensorBoard(log_dir='/media/jsl18/Lexar/exp6',
histogram_freq=1,
write_graph=True,
write_images=False)
nb_hidden = 50
input1 = Input(shape=(1000, ))
input2 = Input(shape=(1000, ))
input3 = Input(shape=(1000, ))
concatL = Concatenate()([input1, input2, input3])
#layer1=Dense(nb_hidden,bias_initializer='zeros')(concatL)
#layer1=LeakyReLU(alpha=0.01)(layer1)
out = Dense(nb_classes, bias_initializer='zeros',
activation='softmax')(concatL)
model = Model([input1, input2, input3], out)
#model=load_model('genFit_ens450.h5');
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy', top3_acc, top5_acc])
G = myGeneratorTrain()
K = next(G)
X_i = K[0]
Y_i = K[1]
#model.fit(X_i,Y_i)
def _scorer(doc_inputs, dataid):
self.visout_count += 1
self.vis_out = {}
doc_qts_scores = [query_idf]
for ng in sorted(ng_fsizes):
if p['distill'] == 'firstk':
input_ng = max(ng_fsizes)
else:
input_ng = ng
for n_x, n_y in ng_fsizes[ng]:
dim_name = self._get_dim_name(n_x, n_y)
if n_x == 1 and n_y == 1:
doc_cov = doc_inputs[input_ng]
re_doc_cov = doc_cov
else:
doc_cov = cov_sim_layers[dim_name](re_input(
doc_inputs[input_ng]))
re_doc_cov = re_lq_ds[dim_name](
pool_filter_layer[dim_name](Permute(
(1, 3, 2))(doc_cov)))
self.vis_out['conv%s' % ng] = doc_cov
if p['context']:
ng_signal = pool_sdim_layer_context[dim_name](
[re_doc_cov, doc_inputs['context']])
else:
ng_signal = pool_sdim_layer[dim_name](re_doc_cov)
doc_qts_scores.append(ng_signal)
if len(doc_qts_scores) == 1:
doc_qts_score = doc_qts_scores[0]
else:
doc_qts_score = Concatenate(axis=2)(doc_qts_scores)
if permute_idxs is not None:
doc_qts_score = Lambda(_permute_scores)(
[doc_qts_score, permute_idxs])
if not p['td']:
# Mode 1
#doc_qts_score = Flatten()(doc_qts_score)
if p['use_bm25']:
doc_qts_score = Concatenate(axis=1)(
[doc_qts_score, doc_inputs['bm25_score']])
if p['use_overlap_features']:
doc_qts_score = Concatenate(axis=1)(
[doc_qts_score, doc_inputs['doc_overlap_features']])
doc_score = rnn_layer(doc_qts_score)
else:
# Mode 2
doc_score = Flatten()(rnn_layer(doc_qts_score))
if p['use_bm25']:
doc_score = Concatenate(axis=1)(
[doc_score, doc_inputs['bm25_score']])
if p['use_overlap_features']:
doc_score = Concatenate(axis=1)(
[doc_score, doc_inputs['doc_overlap_features']])
doc_score = dout_combine(doc_score)
return doc_score
def my_hybrid_network_lanes(inputs,
filters_init_num=16,
data_format="channels_last",
train_backbone=True):
# inputs = Input(shape=input_shape)
filters_num = filters_init_num
kernel_h = 3
kernel_w = 3
kernel = 3
# encoder
conv_1 = Convolution2D(filters_num, (kernel_h, kernel_w),
padding="same",
data_format=data_format,
name='conv_2d_1',
trainable=train_backbone)(inputs)
conv_1 = BatchNormalization(trainable=train_backbone,
name='batch_normalization_1')(conv_1)
conv_1 = Activation("relu")(conv_1)
# conv_1a = Convolution2D(filters_num, (kernel_h, kernel_w), padding="same", data_format=data_format, name='conv_2d_1a',
# trainable=train_backbone)(conv_1)
# conv_1a = BatchNormalization(trainable=train_backbone, name='batch_normalization_1a')(conv_1a)
# conv_1a = Activation("relu")(conv_1a)
#concat_1 = Concatenate()([inputs, pool_1])
pool_1 = MaxPooling2D((2, 2))(conv_1)
filters_num = filters_init_num * 2
conv_2 = Convolution2D(filters_num, (kernel_h, kernel_w),
padding="same",
data_format=data_format,
name='conv_2d_2',
trainable=train_backbone)(pool_1)
conv_2 = BatchNormalization(trainable=train_backbone,
name='batch_normalization_2')(conv_2)
conv_2 = Activation("relu")(conv_2)
# conv_2a = Convolution2D(filters_num, (kernel_h, kernel_w), padding="same", data_format=data_format,name='conv_2d_2a',
# trainable=train_backbone)(conv_2)
# conv_2a = BatchNormalization(trainable=train_backbone, name='batch_normalization_2a')(conv_2a)
# conv_2a = Activation("relu")(conv_2a)
pool_2 = MaxPooling2D((2, 2))(conv_2)
# concat_2 = Concatenate()([conv_1, conv_2])
filters_num = filters_init_num * 4
conv_3 = Convolution2D(
filters_num,
(kernel_h, kernel_w),
padding="same",
data_format=data_format,
trainable=train_backbone,
name='conv_2d_3',
)(pool_2)
conv_3 = BatchNormalization(trainable=train_backbone,
name='batch_normalization_3')(conv_3)
conv_3 = Activation("relu")(conv_3)
# conv_3a = Convolution2D(filters_num, (kernel_h, kernel_w), padding="same", data_format=data_format,
# trainable=train_backbone, name='conv_2d_3a')(conv_3)
# conv_3a = BatchNormalization(trainable=train_backbone, name='batch_normalization_3a')(conv_3a)
# conv_3a = Activation("relu")(conv_3a)
pool_3 = MaxPooling2D((2, 2))(conv_3)
pool_3a = AveragePooling2D((2, 2))(conv_3)
concat_3 = Concatenate()([pool_3a, pool_3])
# flatten_0 = Flatten()(pool_3)
# fc_min1 = Dense(6912, activation='relu', name='fcmin_819')(flatten_0)
# print(pool_3.shape)
# back2bsnes = Reshape((36, 6, 32))(fc_min1)
filters_num = filters_init_num * 8
conv_4 = Convolution2D(filters_num, (kernel_h, kernel_w),
padding="same",
data_format=data_format,
trainable=train_backbone,
name='conv_2d_4')(concat_3)
conv_4 = BatchNormalization(trainable=train_backbone,
name='batch_normalization_4')(conv_4)
conv_4 = Activation("relu")(conv_4)
# conv_4a = Convolution2D(filters_num, (kernel_h, kernel_w), padding="same", data_format=data_format,
# trainable=train_backbone, name='conv_2d_4a')(conv_4)
# conv_4a = BatchNormalization(trainable=train_backbone, name='batch_normalization_4a')(conv_4a)
# conv_4a = Activation("relu")(conv_4a)
pool_4 = MaxPooling2D((2, 2))(conv_4)
pool_4a = AveragePooling2D((2, 2))(conv_4)
concat_4 = Concatenate()([pool_4a, pool_4])
filters_num = filters_init_num * 16
conv_5 = Convolution2D(filters_num, (kernel_h, kernel_w),
padding="same",
data_format=data_format,
trainable=train_backbone,
name='conv_2d_5')(concat_4)
conv_5 = BatchNormalization(trainable=train_backbone,
name='batch_normalization_5')(conv_5)
conv_5 = Activation("relu")(conv_5)
pool_5 = MaxPooling2D((3, 2))(conv_5)
pool_5a = AveragePooling2D((3, 2))(conv_5)
concat_5 = Concatenate()([pool_5a, pool_5])
# # Classification
flatten = Flatten()(concat_5)
# fc_lanes = Dense(819, activation='relu', name='fc_lanes')(flatten)
return flatten
pool2 = MaxPooling1D(pool_size=int(conv2.shape[1]),
strides=stride,
padding='valid')(conv2)
pool2 = Flatten()(pool2)
conv3 = Convolution1D(filters=num_filter,
kernel_size=s3,
padding="valid",
activation="relu",
strides=1)(z)
pool3 = MaxPooling1D(pool_size=int(conv3.shape[1]),
strides=stride,
padding='valid')(conv3)
pool3 = Flatten()(pool3)
fc_1 = Concatenate()([pool1, pool2, pool3])
#fc_1=Dropout(0.5)(fc_1)
out = Dense(class_number)(fc_1)
model_output = Activation('sigmoid')(out)
model = Model(inputs=model_input, outputs=model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights([weights])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=num_epochs,
validation_data=(x_test, y_test))
def AlexNet(weights_path=None, retainTop=True):
inputs = Input(shape=(3, 227, 227))
conv_1 = Conv2D(96, (11, 11),
strides=(4, 4),
activation='relu',
name='conv_1')(inputs)
conv_2 = MaxPooling2D((3, 3), strides=(2, 2))(conv_1)
#pdb.set_trace()
conv_2 = crosschannelnormalization(name='convpool_1')(conv_2)
conv_2 = ZeroPadding2D((2, 2))(conv_2)
concat1 = Concatenate(axis=1, name='conv_2')
conv_2 = concat1([
Conv2D(128, (5, 5), activation='relu',
name='conv_2_' + str(i + 1))(splittensor(ratio_split=2,
id_split=i)(conv_2))
for i in range(2)
])
conv_3 = MaxPooling2D((3, 3), strides=(2, 2))(conv_2)
#conv_3 = crosschannelnormalization()(conv_3)
conv_3 = ZeroPadding2D((1, 1))(conv_3)
conv_3 = Conv2D(384, (3, 3), activation='relu', name='conv_3')(conv_3)
conv_4 = ZeroPadding2D((1, 1))(conv_3)
concat2 = Concatenate(axis=1, name='conv_4')
conv_4 = concat2([
Conv2D(192, (3, 3), activation='relu',
name='conv_4_' + str(i + 1))(splittensor(ratio_split=2,
id_split=i)(conv_4))
for i in range(2)
])
conv_5 = ZeroPadding2D((1, 1))(conv_4)
concat3 = Concatenate(axis=1, name='conv_5')
conv_5 = concat3([
Conv2D(128, (3, 3), activation='relu',
name='conv_5_' + str(i + 1))(splittensor(ratio_split=2,
id_split=i)(conv_5))
for i in range(2)
])
dense_1 = MaxPooling2D((3, 3), strides=(2, 2), name='convpool_5')(conv_5)
dense_1 = Flatten(name='flatten')(dense_1)
dense_1 = Dense(4096, activation='relu', name='dense_1')(dense_1)
dense_2 = Dropout(0.5)(dense_1)
dense_2 = Dense(4096, activation='relu', name='dense_2')(dense_2)
dense_3 = Dropout(0.5)(dense_2)
dense_3 = Dense(1000, name='dense_3')(dense_3)
prediction = Activation('softmax', name='softmax')(dense_3)
tempModel = Model(inputs=[inputs], outputs=[prediction])
tempModel.load_weights(weights_path)
if not retainTop:
model = Model(inputs=[inputs], outputs=[dense_2])
lastLayer = dense_2
else:
model = tempModel
lastLayer = prediction
firstLayer = inputs
return model, firstLayer, lastLayer
weights_path = "keras/model.h5" # orginal weights converted from caffe
#weights_path = "training/weights.best.h5" # weights tarined from scratch
input_shape = (None, None, 3)
img_input = Input(shape=input_shape)
stages = 6
np_branch1 = 38
np_branch2 = 19
img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5]
# VGG
stage0_out = vgg_block(img_normalized)
# stage 1
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out])
# stage t >= 2
for sn in range(2, stages + 1):
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2)
if (sn < stages):
x = Concatenate()([stageT_branch1_out, stageT_branch2_out, stage0_out])
model = Model(img_input, [stageT_branch1_out, stageT_branch2_out])
model.load_weights(weights_path)
def __init__(self, numClasses=6, imageWidth=256, imageHeight=256):
self.classes = {}
self.numClasses = numClasses
self.imageWidth = imageWidth
self.imageHeight = imageHeight
input_image = Input(shape=(self.imageWidth, self.imageHeight, 3))
# ------------------------------------ TOP BRANCH ------------------------------------
# first top convolution layer
top_conv1 = Convolution2D(filters=48, kernel_size=(11, 11), strides=(4, 4),
input_shape=(self.imageWidth, self.imageHeight, 3), activation='relu')(input_image)
top_conv1 = BatchNormalization()(top_conv1)
top_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_conv1)
# second top convolution layer
# split feature map by half
top_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(top_conv1)
top_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(top_conv1)
top_top_conv2 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_top_conv2)
top_top_conv2 = BatchNormalization()(top_top_conv2)
top_top_conv2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_top_conv2)
top_bot_conv2 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_bot_conv2)
top_bot_conv2 = BatchNormalization()(top_bot_conv2)
top_bot_conv2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_bot_conv2)
# third top convolution layer
# concat 2 feature map
top_conv3 = Concatenate()([top_top_conv2, top_bot_conv2])
top_conv3 = Convolution2D(filters=192, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_conv3)
# fourth top convolution layer
# split feature map by half
top_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(top_conv3)
top_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(top_conv3)
top_top_conv4 = Convolution2D(filters=96, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_top_conv4)
top_bot_conv4 = Convolution2D(filters=96, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_bot_conv4)
# fifth top convolution layer
top_top_conv5 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_top_conv4)
top_top_conv5 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_top_conv5)
top_bot_conv5 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(top_bot_conv4)
top_bot_conv5 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(top_bot_conv5)
# ------------------------------------ TOP BOTTOM ------------------------------------
# first bottom convolution layer
bottom_conv1 = Convolution2D(filters=48, kernel_size=(11, 11), strides=(4, 4),
input_shape=(self.imageWidth, self.imageHeight, 3), activation='relu')(input_image)
bottom_conv1 = BatchNormalization()(bottom_conv1)
bottom_conv1 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_conv1)
# second bottom convolution layer
# split feature map by half
bottom_top_conv2 = Lambda(lambda x: x[:, :, :, :24])(bottom_conv1)
bottom_bot_conv2 = Lambda(lambda x: x[:, :, :, 24:])(bottom_conv1)
bottom_top_conv2 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_top_conv2)
bottom_top_conv2 = BatchNormalization()(bottom_top_conv2)
bottom_top_conv2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_top_conv2)
bottom_bot_conv2 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_bot_conv2)
bottom_bot_conv2 = BatchNormalization()(bottom_bot_conv2)
bottom_bot_conv2 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_bot_conv2)
# third bottom convolution layer
# concat 2 feature map
bottom_conv3 = Concatenate()([bottom_top_conv2, bottom_bot_conv2])
bottom_conv3 = Convolution2D(filters=192, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_conv3)
# fourth bottom convolution layer
# split feature map by half
bottom_top_conv4 = Lambda(lambda x: x[:, :, :, :96])(bottom_conv3)
bottom_bot_conv4 = Lambda(lambda x: x[:, :, :, 96:])(bottom_conv3)
bottom_top_conv4 = Convolution2D(filters=96, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_top_conv4)
bottom_bot_conv4 = Convolution2D(filters=96, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_bot_conv4)
# fifth bottom convolution layer
bottom_top_conv5 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_top_conv4)
bottom_top_conv5 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_top_conv5)
bottom_bot_conv5 = Convolution2D(filters=64, kernel_size=(3, 3), strides=(1, 1), activation='relu',
padding='same')(bottom_bot_conv4)
bottom_bot_conv5 = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(bottom_bot_conv5)
# ---------------------------------- CONCATENATE TOP AND BOTTOM BRANCH ------------------------------------
conv_output = Concatenate()([top_top_conv5, top_bot_conv5, bottom_top_conv5, bottom_bot_conv5])
# Flatten
flatten = Flatten()(conv_output)
# Fully-connected layer
FC_1 = Dense(units=4096, activation='relu')(flatten)
FC_1 = Dropout(0.6)(FC_1)
FC_2 = Dense(units=4096, activation='relu')(FC_1)
FC_2 = Dropout(0.6)(FC_2)
output = Dense(units=self.numClasses, activation='softmax')(FC_2)
self.model = Model(inputs=input_image, outputs=output)
sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)
self.model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
z = Embedding(len(vocabulary_inv), embedding_dim, input_length=sequence_length, name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
assert len(Xtrain) == len(Ytrain), 'Difference in len between Xtrain and Ytrain!'
# Finalize model building
model = Model(model_input, model_output)
# Compile model
model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"])
# Initialize weights with word2vec
if model_type == "CNN-non-static":
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape", weights.shape)
def get_training_model(weight_decay, gpus=None):
img_input_shape = (None, None, 3)
vec_input_shape = (None, None, 38)
heat_input_shape = (None, None, 19)
inputs = []
outputs = []
img_input = Input(shape=img_input_shape)
vec_weight_input = Input(shape=vec_input_shape)
heat_weight_input = Input(shape=heat_input_shape)
inputs.append(img_input)
inputs.append(vec_weight_input)
inputs.append(heat_weight_input)
img_normalized = Lambda(lambda x: x / 256 - 0.5)(img_input) # [-0.5, 0.5]
# VGG
stage0_out = vgg_block(img_normalized, weight_decay)
# stage 1 - branch 1 (PAF)
stage1_branch1_out = stage1_block(stage0_out, np_branch1, 1, weight_decay)
w1 = apply_mask(stage1_branch1_out, vec_weight_input, heat_weight_input,
np_branch1, 1, 1)
# stage 1 - branch 2 (confidence maps)
stage1_branch2_out = stage1_block(stage0_out, np_branch2, 2, weight_decay)
w2 = apply_mask(stage1_branch2_out, vec_weight_input, heat_weight_input,
np_branch2, 1, 2)
x = Concatenate()([stage1_branch1_out, stage1_branch2_out, stage0_out])
outputs.append(w1)
outputs.append(w2)
# stage sn >= 2
for sn in range(2, stages + 1):
# stage SN - branch 1 (PAF)
stageT_branch1_out = stageT_block(x, np_branch1, sn, 1, weight_decay)
w1 = apply_mask(stageT_branch1_out, vec_weight_input,
heat_weight_input, np_branch1, sn, 1)
# stage SN - branch 2 (confidence maps)
stageT_branch2_out = stageT_block(x, np_branch2, sn, 2, weight_decay)
w2 = apply_mask(stageT_branch2_out, vec_weight_input,
heat_weight_input, np_branch2, sn, 2)
outputs.append(w1)
outputs.append(w2)
if (sn < stages):
x = Concatenate()(
[stageT_branch1_out, stageT_branch2_out, stage0_out])
if gpus is None:
model = Model(inputs=inputs, outputs=outputs)
else:
import tensorflow as tf
with tf.device('/cpu:0'
): #this model will not be actually used, it's template
model = Model(inputs=inputs, outputs=outputs)
return model
def build_nn_model(self, element_dim=103,
conv_window=3, conv_filters=64,
rnn_dim=64, recipe_latent_dim=8,
intermediate_dim=64, latent_dim=8,
max_material_length=10, charset_size=50,):
self.latent_dim = latent_dim
self.recipe_latent_dim = recipe_latent_dim
self.original_dim = max_material_length * charset_size
x_mat = Input(shape=(max_material_length, charset_size), name="material_in")
conv_x1 = Conv1D(conv_filters, conv_window, padding="valid", activation="relu", name='conv_enc_1')(x_mat)
conv_x2 = Conv1D(conv_filters, conv_window, padding="valid", activation="relu", name='conv_enc_2')(conv_x1)
conv_x3 = Conv1D(conv_filters, conv_window, padding="valid", activation="relu", name='conv_enc_3')(conv_x2)
h_flatten = Flatten()(conv_x3)
h = Dense(intermediate_dim, activation="relu", name="hidden_enc")(h_flatten)
z_mean_func = Dense(latent_dim, name="means_enc")
z_log_var_func = Dense(latent_dim, name="vars_enc")
z_mean = z_mean_func(h)
z_log_var = z_log_var_func(h)
def sample(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(latent_dim,), mean=0.0, stddev=1.0)
return z_mean + K.exp(z_log_var / 2) * epsilon
z = Lambda(sample, name="lambda_sample")([z_mean, z_log_var])
c_element = Input(shape=(element_dim,), name="cond_element_in")
c_latent_recipe = Input(shape=(recipe_latent_dim,), name="cond_latent_recipe_in")
z_conditional = Concatenate(name="concat_cond")([z, c_latent_recipe, c_element])
decoder_h = Dense(intermediate_dim, activation="relu", name="hidden_dec")
decoder_h_repeat = RepeatVector(max_material_length, name="h_rep_dec")
decoder_h_gru_1 = GRU(rnn_dim, return_sequences=True, name="recurrent_dec_1")
decoder_h_gru_2 = GRU(rnn_dim, return_sequences=True, name="recurrent_dec_2")
decoder_h_gru_3 = GRU(rnn_dim, return_sequences=True, name="recurrent_dec_3")
decoder_mat = TimeDistributed(Dense(charset_size, activation='softmax'), name="means_material_dec")
h_decoded = decoder_h(z_conditional)
h_decode_repeat = decoder_h_repeat(h_decoded)
gru_h_decode_1 = decoder_h_gru_1(h_decode_repeat)
gru_h_decode_2 = decoder_h_gru_2(gru_h_decode_1)
gru_h_decode_3 = decoder_h_gru_3(gru_h_decode_2)
x_decoded_mat = decoder_mat(gru_h_decode_3)
def vae_xent_loss(x, x_decoded_mean):
x = K.flatten(x)
x_decoded_mean = K.flatten(x_decoded_mean)
rec_loss = self.original_dim * metrics.binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.mean(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return rec_loss + kl_loss
encoder = Model(inputs=[x_mat], outputs=[z_mean])
decoder_x_input = Input(shape=(latent_dim,))
if use_conditionals:
decoder_inputs = Concatenate(name="concat_cond_dec")([decoder_x_input, c_latent_recipe, c_element])
else:
decoder_inputs = decoder_x_input
_h_decoded = decoder_h(decoder_inputs)
_h_decode_repeat = decoder_h_repeat(_h_decoded)
_gru_h_decode_1 = decoder_h_gru_1(_h_decode_repeat)
_gru_h_decode_2 = decoder_h_gru_2(_gru_h_decode_1)
_gru_h_decode_3 = decoder_h_gru_3(_gru_h_decode_2)
_x_decoded_mat = decoder_mat(_gru_h_decode_3)
decoder = Model(inputs=[decoder_x_input, c_latent_recipe, c_element],
outputs=[_x_decoded_mat])
vae = Model(inputs=[x_mat, c_latent_recipe, c_element],
outputs=[x_decoded_mat])
vae.compile(
optimizer=Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=True),
loss=vae_xent_loss,
metrics=['categorical_accuracy']
)
self.vae = vae
self.encoder = encoder
self.decoder = decoder
#2-2. 모델 구성 input1 = Input(shape=(5, 1)) x1 = LSTM(10)(input1) x1 = Dense(700)(x1) x1 = Dense(500)(x1) x1 = Dense(300)(x1) x1 = Dropout(0.2)(x1) x1 = Dense(100)(x1) input2 = Input(shape=(6, 1)) x2 = LSTM(100)(input2) x2 = Dense(20)(x2) x2 = Dense(700)(x2) x2 = Dense(500)(x2) x2 = Dense(300)(x2) x2 = Dropout(0.2)(x2) x2 = Dense(100)(x2) merge = Concatenate()([x1, x2]) output = Dense(1)(merge) model = Model(inputs=[input1, input2], outputs=output) # model.summary() #3. 컴파일, 훈련 model.compile(loss='mse', optimizer='adam', metrics=['mse']) model.fit([x_sam, x_hite], y_sam, epochs=100)
def create_model_hierarchy(cls,
bottom_item_list,
emb_wgts_bottom_items_dict,
layer_nums=3,
rnn_state_size=[],
bottom_emb_item_len=3,
flag_embedding_trainable=1,
seq_len=39,
batch_size=20,
mode_attention=1,
drop_out_r=0.,
att_layer_cnt=2,
bhDwellAtt=0,
rnn_type="WGRU",
RNN_norm="GRU",
flagCid3RNNNorm=False):
c_mask_value = 0.
att_zero_value = -2 ^ 31
def slice(x):
return x[:, -1, :]
flag_concate_sku_cid = True
RNN = rnn_type
MODE_BHDWELLATT = True if bhDwellAtt == 1 else False
ATT_NET_LAYER_CNT = att_layer_cnt
bottom_item_len = len(bottom_item_list)
input = [None] * bottom_item_len
word_num = [None] * bottom_item_len
emb_len = [None] * bottom_item_len
embedding_bottom_item = [None] * bottom_item_len
embed = [None] * bottom_item_len
layer_nums_max = 3
rnn_embed = [None] * layer_nums_max
rnn = [None] * layer_nums_max
rnn_output = [None] * layer_nums_max
flag_embedding_trainable = True if flag_embedding_trainable == 1 else False
##Embedding layer
# Embedding sku, bh, cid3, gap, dwell: 0, 1, 2, 3, 4
for i in range(bottom_item_len):
bottom_item = bottom_item_list[i]
###input
input[i] = Input(batch_shape=(
batch_size,
seq_len,
),
dtype='int32')
###Embedding
# load embedding weights
# emb_wgts[i] = np.loadtxt(init_wgts_file_emb[i])
word_num[i], emb_len[i] = emb_wgts_bottom_items_dict[
bottom_item].shape
print
word_num[i], emb_len[i]
# get embedding
cur_flag_embedding_trainable = flag_embedding_trainable
if (i == 0):
cur_flag_embedding_trainable = False
embedding_bottom_item[i] = Embedding(
word_num[i],
emb_len[i],
input_length=seq_len,
trainable=cur_flag_embedding_trainable)
embed[i] = embedding_bottom_item[i](input[i]) # drop_out=0.2
embedding_bottom_item[i].set_weights(
[emb_wgts_bottom_items_dict[bottom_item]])
# cal mask
mask_sku = np.zeros((batch_size, seq_len))
mask_cid3 = np.zeros((batch_size, seq_len))
for j in range(batch_size):
sku = input[0][j, :]
cid3 = input[2][j, :]
for k in range(seq_len - 1):
if (sku[k] == 0 or sku[k] == sku[k + 1]):
mask_sku[j][k] = 1
if (sku[k] == 0 or cid3[k] == cid3[k + 1]):
mask_cid3[j][k] = 1
# f mask
def f_mask_sku(x):
x_new = x
for j in range(batch_size):
for k in range(seq_len):
if (mask_sku[j][k] == 1):
x_new = T.set_subtensor(x_new[j, k, :], c_mask_value)
return x_new
def f_mask_cid3(x):
x_new = x
for j in range(batch_size):
for k in range(seq_len):
if (mask_cid3[j][k] == 1):
x_new = T.set_subtensor(x_new[j, k, :], c_mask_value)
return x_new
def f_mask_att_sku(x):
x_new = x
for j in range(batch_size):
for k in range(seq_len):
if (mask_sku[j][k] == 1):
x_new = T.set_subtensor(x_new[j, k], att_zero_value)
return x_new
def f_mask_att_cid3(x):
x_new = x
for j in range(batch_size):
for k in range(seq_len):
if (mask_cid3[j][k] == 1):
x_new = T.set_subtensor(x_new[j, k], att_zero_value)
return x_new
def K_dot(arr):
axes = [1, 1]
x, y = arr[0], arr[1]
return K.batch_dot(x, y, axes=axes)
def K_squeeze(x):
return K.squeeze(x, axis=-1)
Lambda_sequeeze = Lambda(lambda x: K_squeeze(x))
##RNN layer
if (RNN == "BLSTM"):
rnn[0] = BLSTM(rnn_state_size[0],
interval_dim=emb_len[3],
weight_dim=emb_len[1],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_micro")
rnn[1] = BLSTM(rnn_state_size[1],
interval_dim=emb_len[3],
weight_dim=emb_len[4],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_sku")
if (not flagCid3RNNNorm):
rnn[2] = BLSTM(rnn_state_size[2],
interval_dim=emb_len[3],
weight_dim=0,
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_cid3")
elif (RNN == "BLSTM2"):
rnn[0] = BLSTM2(rnn_state_size[0],
interval_dim=emb_len[3],
weight_dim=emb_len[1],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_micro")
rnn[1] = BLSTM2(rnn_state_size[1],
interval_dim=emb_len[3],
weight_dim=emb_len[4],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_sku")
if (not flagCid3RNNNorm):
rnn[2] = BLSTM2(rnn_state_size[2],
interval_dim=emb_len[3],
weight_dim=0,
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_cid3")
elif (RNN == "TimeLSTM"):
rnn[0] = BLSTM(rnn_state_size[0],
interval_dim=emb_len[3],
weight_dim=0,
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_micro")
rnn[1] = BLSTM(rnn_state_size[1],
interval_dim=emb_len[3],
weight_dim=0,
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_sku")
if (not flagCid3RNNNorm):
rnn[2] = BLSTM(rnn_state_size[2],
interval_dim=emb_len[3],
weight_dim=0,
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_cid3")
elif (RNN == "WGRU"):
rnn[0] = WGRU(rnn_state_size[0],
weight_dim=emb_len[1],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_micro")
rnn[1] = WGRU(rnn_state_size[1],
weight_dim=emb_len[3],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_sku")
if (not flagCid3RNNNorm):
rnn[2] = WGRU(rnn_state_size[2],
weight_dim=emb_len[3],
tateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_cid3")
elif (RNN == "LSTM" or RNN == "GRU"):
RNN = LSTM if RNN == "LSTM" else GRU
rnn[0] = RNN(rnn_state_size[0],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_micro")
rnn[1] = RNN(rnn_state_size[1],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_sku")
else:
print
"%s is not valid RNN!" % RNN
if (RNN_norm == "LSTM"):
rnn_cid3 = LSTM
else:
rnn_cid3 = GRU
if (flagCid3RNNNorm):
rnn[2] = rnn_cid3(rnn_state_size[2],
stateful=False,
return_sequences=True,
dropout=drop_out_r,
name="rnn_out_cid3")
# rnn embed 0
if (bottom_emb_item_len == 5):
rnn_embed[0] = Concatenate(axis=-1)(
[embed[0], embed[1], embed[2], embed[3], embed[4]])
elif (bottom_emb_item_len == 4):
rnn_embed[0] = Concatenate(axis=-1)(
[embed[0], embed[1], embed[2], embed[3]])
elif (bottom_emb_item_len == 3):
rnn_embed[0] = Concatenate(axis=-1)([embed[0], embed[1], embed[2]])
elif (bottom_emb_item_len == 1):
rnn_embed[0] = embed[0]
elif (bottom_emb_item_len == 2):
rnn_embed[0] = Concatenate(axis=-1)([embed[0], embed[2]])
else:
rnn_embed[0] = Concatenate(axis=-1)(
[embed[0], embed[1], embed[2], embed[3]])
# add interval, wei
if (RNN == "WGRU"):
rnn_embed[0] = Concatenate(axis=-1)([rnn_embed[0], embed[1]])
if (RNN == "BLSTM" or RNN == "BLSTM2"):
rnn_embed[0] = Concatenate(axis=-1)(
[rnn_embed[0], embed[3], embed[1]])
if (RNN == "TimeLSTM"):
rnn_embed[0] = Concatenate(axis=-1)([rnn_embed[0], embed[3]])
# rnn micro
rnn_output[0] = rnn[0](rnn_embed[0])
# rnn sku
if (flag_concate_sku_cid):
rnn_embed[1] = Concatenate(axis=-1)([embed[0], rnn_output[0]])
else:
rnn_embed[1] = rnn_output[0]
# mask sku
# rnn embed 1
# rnn_embed[1] = Lambda(f_mask_sku, output_shape=(seq_len, rnn_state_size[1]))(rnn_embed[1])
if (RNN == "WGRU"):
rnn_embed[1] = Concatenate(axis=-1)([rnn_embed[1], embed[4]])
if (RNN == "BLSTM" or RNN == "BLSTM2"):
rnn_embed[1] = Concatenate(axis=-1)(
[rnn_embed[1], embed[3], embed[4]])
if (RNN == "TimeLSTM"):
rnn_embed[1] = Concatenate(axis=-1)([rnn_embed[1], embed[3]])
rnn_embed[1] = Lambda(f_mask_sku)(rnn_embed[1])
rnn_embed[1] = Masking(mask_value=c_mask_value)(rnn_embed[1])
rnn_output[1] = rnn[1](rnn_embed[1])
# rnn cid3
if (flag_concate_sku_cid):
rnn_embed[2] = Concatenate()([embed[2], rnn_output[1]])
else:
rnn_embed[2] = rnn_output[1]
if (not flagCid3RNNNorm):
rnn_embed[2] = Concatenate(axis=-1)([rnn_embed[2], embed[3]])
# mask cid3
# rnn_embed[2] = Lambda(f_mask_cid3, output_shape=(seq_len, rnn_state_size[2]))(rnn_embed[2])
rnn_embed[2] = Lambda(f_mask_cid3)(rnn_embed[2])
rnn_embed[2] = Masking(mask_value=c_mask_value)(rnn_embed[2])
rnn_output[2] = rnn[2](rnn_embed[2])
# rnn final output
rnn_out_final = rnn_output[layer_nums - 1]
rnn_out_micro = rnn_output[0]
rnn_out_sku = rnn_output[1]
rnn_out_cid3 = rnn_output[2]
# predict sku, cid3
if (mode_attention == 0):
# micro
att_out_micro = Lambda(
slice, output_shape=(rnn_state_size[0], ))(rnn_out_micro)
# trans to sku emb len
out_micro_sku_emb = Dense(emb_len[0],
activation="tanh")(att_out_micro)
out_micro = out_micro_sku_emb
# sku
att_out_sku = Lambda(
slice, output_shape=(rnn_state_size[1], ))(rnn_out_sku)
# trans to sku emb len
out_sku_emb = Dense(emb_len[0], activation="tanh")(att_out_sku)
out_sku = out_sku_emb
# cid3
att_out_cid3 = Lambda(
slice, output_shape=(rnn_state_size[2], ))(rnn_out_cid3)
out_cid3_emb = Dense(emb_len[2], activation="tanh")(att_out_cid3)
out_cid3 = out_cid3_emb
# out_cid3 = Dense(word_num[2], activation="softmax")(out_cid3_emb)
if (mode_attention == 2):
# atten micro
m_h = rnn_out_micro
m_h_last = Lambda(slice,
output_shape=(rnn_state_size[0], ),
name="rnn_out_micro_last")(m_h)
m_h_r = RepeatVector(seq_len)(m_h_last)
if (MODE_BHDWELLATT):
m_h_c = Concatenate(axis=-1)([m_h, m_h_r, embed[1]])
else:
m_h_c = Concatenate(axis=-1)([m_h, m_h_r])
if (ATT_NET_LAYER_CNT == 2):
m_h_a_1 = TimeDistributed(
Dense(ATT_NET_HIDSIZE, activation='tanh'))(m_h_c)
m_h_a = TimeDistributed(Dense(1, activation='tanh'))(m_h_a_1)
else:
m_h_a = TimeDistributed(Dense(1, activation='tanh'))(m_h_c)
m_h_a = Lambda(lambda x: x, output_shape=lambda s: s)(m_h_a)
m_att = Flatten()(m_h_a)
m_att_micro = Softmax(name="att_micro")(m_att)
m_att_out = Lambda(K_dot,
output_shape=(rnn_state_size[0], ),
name="out_micro_pre")([m_h, m_att_micro])
# trans to sku emb len
out_micro = Dense(emb_len[0], activation="tanh")(m_att_out)
# attenion sku
s_h = rnn_out_sku
s_h_last = Lambda(slice,
output_shape=(rnn_state_size[1], ),
name="rnn_out_sku_last")(s_h)
s_h_r = RepeatVector(seq_len)(s_h_last)
if (MODE_BHDWELLATT):
s_h_c = Concatenate(axis=-1)([s_h, s_h_r, embed[4]])
else:
s_h_c = Concatenate(axis=-1)([s_h, s_h_r])
if (ATT_NET_LAYER_CNT == 2):
s_h_a_1 = TimeDistributed(
Dense(ATT_NET_HIDSIZE, activation='tanh'))(s_h_c)
s_h_a = TimeDistributed(Dense(1, activation='tanh'))(s_h_a_1)
else:
s_h_a = TimeDistributed(Dense(1, activation='tanh'))(s_h_c)
s_h_a = Lambda(lambda x: x, output_shape=lambda s: s)(s_h_a)
s_att = Flatten()(s_h_a)
s_att = Lambda(f_mask_att_sku)(s_att)
s_att_sku = Softmax(axis=-1, name="att_sku")(s_att)
s_att_out = Lambda(K_dot,
output_shape=(rnn_state_size[1], ),
name="out_sku_pre")([s_h, s_att_sku])
# attention cid3
c_h = rnn_out_cid3
c_h_last = Lambda(slice,
output_shape=(rnn_state_size[2], ),
name="rnn_out_cid3_last")(c_h)
c_h_r = RepeatVector(seq_len)(c_h_last)
c_h_c = Concatenate(axis=-1)([c_h, c_h_r])
if (ATT_NET_LAYER_CNT == 2):
c_h_a_1 = TimeDistributed(
Dense(ATT_NET_HIDSIZE, activation='tanh'))(c_h_c)
c_h_a = TimeDistributed(Dense(1, activation='tanh'))(c_h_a_1)
else:
c_h_a = TimeDistributed(Dense(1, activation='tanh'))(c_h_c)
c_h_a = Lambda(lambda x: x, output_shape=lambda s: s)(c_h_a)
c_att = Flatten()(c_h_a)
c_att = Lambda(f_mask_att_cid3)(c_att)
c_att_cid3 = Softmax(axis=-1, name="att_cid3")(c_att)
c_att_out = Lambda(K_dot,
output_shape=(rnn_state_size[2], ),
name="out_cid3_pre")([c_h, c_att_cid3])
out_cid3 = Dense(emb_len[2], activation="tanh")(c_att_out)
out_sku = Dense(emb_len[0], activation="tanh")(s_att_out)
# model
model = Model(
inputs=[input[0], input[1], input[2], input[3], input[4]],
outputs=[out_micro, out_sku, out_cid3])
# return embedding, rnn, ret_with_target, input, out
return model
def __init__(self, n_in, n_out, state_bounds, action_bounds, reward_bound, settings_):
super(DeepCNNKeras,self).__init__(n_in, n_out, state_bounds, action_bounds, reward_bound, settings_)
### Apparently after the first layer the patch axis is left out for most of the Keras stuff...
input = Input(shape=(self._state_length,))
input.trainable = True
print ("Input ", input)
## Custom slice layer, Keras does not have this layer type...
taskFeatures = Lambda(lambda x: x[:,0:self._settings['num_terrain_features']], output_shape=(self._settings['num_terrain_features'],))(input)
# taskFeatures = Lambda(lambda x: x[:,0:self._settings['num_terrain_features']])(input)
characterFeatures = Lambda(lambda x: x[:,self._settings['num_terrain_features']:self._state_length], output_shape=(self._state_length-self._settings['num_terrain_features'],))(input)
# characterFeatures = Reshape((-1, self._state_length-self._settings['num_terrain_features']))(characterFeatures)
# characterFeatures = Lambda(lambda x: x[:,self._settings['num_terrain_features']:self._state_length], output_shape=(1,))(input)
# print("TaskFeature shape: ", taskFeatures.output_shape)
network = Reshape((self._settings['num_terrain_features'], 1))(taskFeatures)
network = Conv1D(filters=16, kernel_size=8)(network)
network = Activation('relu')(network)
network = Conv1D(filters=16, kernel_size=8)(network)
network = Activation('relu')(network)
self._critic_task_part = network
network = Flatten()(network)
# characterFeatures = Flatten()(characterFeatures)
# network = Concatenate(axis=1)([network, characterFeatures])
## network.shape should == (None, self._settings['num_terrain_features']) and
## characterFeatures.shape should == (None, state_length-self._settings['num_terrain_features'])
print ("characterFeatures ", characterFeatures)
print ("network ", network)
network = Concatenate(axis=1)([network, characterFeatures])
network = Dense(128, init='uniform')(network)
print ("Network: ", network)
network = Activation('relu')(network)
network = Dense(64, init='uniform')(network)
network = Activation('relu')(network)
network = Dense(32, init='uniform')(network)
network = Activation('relu')(network)
# 1 output, linear activation
network = Dense(1, init='uniform')(network)
network = Activation('linear')(network)
self._critic = Model(input=input, output=network)
inputAct = Input(shape=(self._state_length, ))
inputAct.trainable = True
print ("Input ", inputAct)
## Custom slice layer, Keras does not have this layer type...
taskFeaturesAct = Lambda(lambda x: x[:,0:self._settings['num_terrain_features']], output_shape=(self._settings['num_terrain_features'],))(inputAct)
characterFeaturesAct = Lambda(lambda x: x[:,self._settings['num_terrain_features']:self._state_length], output_shape=(self._state_length-self._settings['num_terrain_features'], ))(inputAct)
## Keras/Tensorflow likes the channels to be at the end
networkAct = Reshape((self._settings['num_terrain_features'], 1))(taskFeaturesAct)
networkAct = Conv1D(filters=16, kernel_size=8)(networkAct)
networkAct = Activation('relu')(networkAct)
networkAct = Conv1D(filters=16, kernel_size=8)(networkAct)
networkAct = Activation('relu')(networkAct)
self._actor_task_part = networkAct
networkAct = Flatten()(networkAct)
networkAct = Concatenate(axis=1)([networkAct, characterFeaturesAct])
networkAct = Dense(128, init='uniform')(networkAct)
print ("Network: ", networkAct)
networkAct = Activation('relu')(networkAct)
networkAct = Dense(64, init='uniform')(networkAct)
networkAct = Activation('relu')(networkAct)
networkAct = Dense(32, init='uniform')(networkAct)
networkAct = Activation('relu')(networkAct)
# 1 output, linear activation
networkAct = Dense(self._action_length, init='uniform')(networkAct)
networkAct = Activation('linear')(networkAct)
self._actor = Model(input=inputAct, output=networkAct)
def create_model(input_shape=(SHAPE_Y, SHAPE_X, 3)):
n_layers_down = [2, 2, 2]
n_layers_up = [2, 2, 2]
n_filters_down = [5, 8, 10]
n_filters_up = [4, 8, 10]
kernels_up = [(3, 3), (3, 3), (3, 3), (3, 3)]
n_filters_center = 6
n_layers_center = 4
print('n_filters_down:%s n_layers_down:%s' %
(str(n_filters_down), str(n_layers_down)))
print('n_filters_center:%d n_layers_center:%d' %
(n_filters_center, n_layers_center))
print('n_filters_up:%s n_layers_up:%s' %
(str(n_filters_up), str(n_layers_up)))
activation = 'relu'
input1 = Input(shape=input_shape)
input2 = Input(shape=input_shape)
inputs = Concatenate()([input1, input2])
x = inputs
x = BatchNormalization()(x)
xbn = x
depth = 0
back_links = []
for n_filters in n_filters_down:
n_layers = n_layers_down[depth]
x, down_link = unet_down_1(n_filters,
x,
activation=activation,
pool=None,
n_layers=n_layers)
back_links.append(down_link)
depth += 1
#x2=Lambda(div_into_patch_back,output_shape=K.get_variable_shape(x)[1:])(x)
x2 = x
center, _ = unet_down_1(n_filters_center,
x2,
activation='relu',
pool=None,
n_layers=n_layers_center)
# center
x3 = center
while depth > 0:
depth -= 1
link = back_links.pop()
n_filters = n_filters_up[depth]
n_layers = n_layers_up[depth]
x3 = unet_up_1(n_filters,
x3,
link,
activation='relu',
n_layers=n_layers)
#if depth <= 1:
#x1 = Dropout(0.25)(x1)
#x1 = concatenate([x1,xbn])
x3 = Conv2D(4, (3, 3), padding='same', activation=None)(x3)
x3 = BatchNormalization()(x3)
F = Conv2D(2, (1, 1), activation=None)(x3)
model = Model(inputs=[input1, input2], outputs=[F])
# model.compile(loss=c_recMse_grad,optimizer="Adam")
return model
def transformer_model(conditioning_input_shapes, conditioning_input_names=None,
output_shape=None,
model_name='CVAE_transformer',
transform_latent_shape=(100,),
transform_type=None,
color_transform_type=None,
enc_params=None,
condition_on_image=True,
n_concat_scales=3,
transform_activation=None, clip_output_range=None,
source_input_idx=None,
mask_by_conditioning_input_idx=None,
):
# collect conditioning inputs, and concatentate them into a stack
if not isinstance(conditioning_input_shapes, list):
conditioning_input_shapes = [conditioning_input_shapes]
if conditioning_input_names is None:
conditioning_input_names = ['cond_input_{}'.format(ii) for ii in range(len(conditioning_input_shapes))]
conditioning_inputs = []
for ii, input_shape in enumerate(conditioning_input_shapes):
conditioning_inputs.append(Input(input_shape, name=conditioning_input_names[ii]))
conditioning_input_stack = Concatenate(name='concat_cond_inputs', axis=-1)(conditioning_inputs)
conditioning_input_shape = tuple(conditioning_input_stack.get_shape().as_list()[1:])
n_dims = len(conditioning_input_shape) - 1
# we will always give z as a flattened vector
z_input = Input((np.prod(transform_latent_shape),), name='z_input')
# determine what we should apply the transformation to
if source_input_idx is None:
# the image we want to transform is exactly the input of the conditioning branch
x_source = conditioning_input_stack
source_input_shape = conditioning_input_shape
else:
# slice conditioning input to get the single source im that we will apply the transform to
source_input_shape = conditioning_input_shapes[source_input_idx]
x_source = conditioning_inputs[source_input_idx]
# assume the output is going to be the transformed source input, so it should be the same shape
if output_shape is None:
output_shape = source_input_shape
if mask_by_conditioning_input_idx is None:
x_mask = None
else:
print('Masking output by input {} with name {}'.format(
mask_by_conditioning_input_idx,
conditioning_input_names[mask_by_conditioning_input_idx]
))
mask_shape = conditioning_input_shapes[mask_by_conditioning_input_idx]
x_mask = conditioning_inputs[mask_by_conditioning_input_idx]
if transform_type == 'flow':
layer_prefix = 'flow'
elif transform_type == 'color':
layer_prefix = 'color'
else:
layer_prefix = 'synth'
if condition_on_image: # assume we always condition by concat since it's better than other forms
# simply concatenate the conditioning stack (at various scales) with the decoder volumes
include_fullres = True
concat_decoder_outputs_with = [None] * len(enc_params['nf_dec'])
concat_skip_sizes = [None] * len(enc_params['nf_dec'])
# make sure x_I is the same shape as the output, including in the channels dimension
if not np.all(output_shape <= conditioning_input_shape):
tile_factor = [int(round(output_shape[i] / conditioning_input_shape[i])) for i in
range(len(output_shape))]
print('Tile factor: {}'.format(tile_factor))
conditioning_input_stack = Lambda(lambda x: tf.tile(x, [1] + tile_factor), name='lambda_tile_cond_input')(conditioning_input_stack)
# downscale the conditioning inputs by the specified number of times
xs_downscaled = [conditioning_input_stack]
for si in range(n_concat_scales):
curr_x_scaled = network_layers.Blur_Downsample(
n_chans=conditioning_input_shape[-1], n_dims=n_dims,
do_blur=True,
name='downsample_scale-1/{}'.format(2**(si + 1))
)(xs_downscaled[-1])
xs_downscaled.append(curr_x_scaled)
if not include_fullres:
xs_downscaled = xs_downscaled[1:] # exclude the full-res volume
print('Including downsampled input sizes {}'.format([x.get_shape().as_list() for x in xs_downscaled]))
concat_decoder_outputs_with[:len(xs_downscaled)] = list(reversed(xs_downscaled))
concat_skip_sizes[:len(xs_downscaled)] = list(reversed(
[np.asarray(x.get_shape().as_list()[1:-1]) for x in xs_downscaled if
x is not None]))
else:
# just ignore the conditioning input
concat_decoder_outputs_with = None
concat_skip_sizes = None
if 'ks' not in enc_params:
enc_params['ks'] = 3
if not enc_params['fully_conv']:
# determine what size to reshape the latent vector to
reshape_encoding_to = basic_networks.get_encoded_shape(
img_shape=conditioning_input_shape,
conv_chans=enc_params['nf_enc'],
)
if np.all(reshape_encoding_to[:n_dims] > concat_skip_sizes[-1][:n_dims]):
raise RuntimeWarning(
'Attempting to concatenate reshaped latent vector of shape {} with downsampled input of shape {}!'.format(
reshape_encoding_to,
concat_skip_sizes[-1]
))
x_enc = Dense(np.prod(reshape_encoding_to))(z_input)
else:
# latent representation is already in correct shape
reshape_encoding_to = transform_latent_shape
x_enc = z_input
x_enc = Reshape(reshape_encoding_to)(x_enc)
print('Decoder starting shape: {}'.format(reshape_encoding_to))
x_transformation = basic_networks.decoder(
x_enc, output_shape,
encoded_shape=reshape_encoding_to,
prefix='{}_dec'.format(layer_prefix),
conv_chans=enc_params['nf_dec'], ks=enc_params['ks'],
n_convs_per_stage=enc_params['n_convs_per_stage'],
use_upsample=enc_params['use_upsample'],
include_skips=concat_decoder_outputs_with,
target_vol_sizes=concat_skip_sizes
)
if transform_activation is not None:
x_transformation = Activation(
transform_activation,
name='activation_transform_{}'.format(transform_activation))(x_transformation)
if transform_type == 'color' and 'delta' in color_transform_type and transform_activation=='tanh':
# TODO: maybe move this logic
# if we are learning a colro delta with a tanh, make sure to multiply it by 2
x_transformation = Lambda(lambda x: x * 2, name='lambda_scale_tanh')(x_transformation)
if mask_by_conditioning_input_idx is not None:
x_transformation = Multiply(name='mult_mask_transformation')([x_transformation, x_mask])
if transform_type is not None:
im_out, transform_out = apply_transformation(x_source, x_transformation,
output_shape=source_input_shape, conditioning_input_shape=conditioning_input_shape, transform_name=transform_type,
apply_flow_transform=transform_type=='flow',
apply_color_transform=transform_type=='color',
color_transform_type=color_transform_type
)
else:
im_out = x_transformation
if clip_output_range is not None:
im_out = Lambda(lambda x: tf.clip_by_value(x, clip_output_range[0], clip_output_range[1]),
name='lambda_clip_output_{}-{}'.format(clip_output_range[0], clip_output_range[1]))(im_out)
if transform_type is not None:
return Model(inputs=conditioning_inputs + [z_input], outputs=[im_out, transform_out], name=model_name)
else:
return Model(inputs=conditioning_inputs + [z_input], outputs=[im_out], name=model_name)
