def __get_model(self, load_weights = False):
stddev = 1.0 / self.vector_dim
initializer = TruncatedNormal(mean=0.0, stddev=stddev, seed=None)
optimizer = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
input_target = Input((1,))
input_context = Input((1,))
embedding = Embedding(self.vocabulary_size, self.vector_dim, input_length=1, name='embedding',
embeddings_initializer=initializer)
target = embedding(input_target)
context = embedding(input_context)
# setup a cosine similarity operation which will be output in a secondary model
similarity = dot([target, context], normalize=True, axes=-1)#we can see that we don't use the similarity for anything but checking
# now perform the dot product operation to get a similarity measure
dot_product = dot([target, context], normalize=False, axes=-1)
dot_product = Reshape((1,), input_shape=(1,1))(dot_product)
# add the sigmoid output layer
output = Dense(1, activation='sigmoid')(dot_product)
# create the primary training model
model = Model(inputs=[input_target, input_context], outputs=output)
model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
# create a secondary validation model to run our similarity checks during training
validation_model = Model(inputs=[input_target, input_context], outputs=similarity)
return model, validation_model
def agent_network(m_bits, k_bits, name):
a_input_0 = Input(shape=(m_bits,)) # message
a_input_1 = Input(shape=(k_bits,)) # key for agent_1
# we use functional writing
a_input = concatenate([a_input_0, a_input_1], axis=1)
###Neural Network architecture
a_dense1 = Dense(units=(m_bits + k_bits))(a_input)
a_dense1a = Activation('tanh')(a_dense1)
a_reshape = Reshape((m_bits + k_bits, 1,))(a_dense1a)
a_conv1 = Conv1D(filters=2, kernel_size=4, strides=1, padding=pad)(a_reshape)
a_conv1a = Activation('tanh')(a_conv1)
a_conv2 = Conv1D(filters=4, kernel_size=4, strides=2, padding=pad)(a_conv1a)
a_conv2a = Activation('tanh')(a_conv2)
a_conv3 = Conv1D(filters=4, kernel_size=1, strides=1, padding=pad)(a_conv2a)
a_conv3a = Activation('tanh')(a_conv3)
a_conv4 = Conv1D(filters=1, kernel_size=1, strides=1, padding=pad)(a_conv3a)
a_conv4a = Activation('sigmoid')(a_conv4)
a_output = Flatten()(a_conv4a)
model = Model([a_input_0, a_input_1], a_output, name=name)
return a_output, model
def create_model(self):
### Create inputs
features = 3
x = Input(shape=(features,))
shared = Dense(units=4 * features)(x)
## task specific network
for count in range(self.__num_task):
task_sepeciic = Dense(units=64, activation='relu')(shared)
att_lstm_inputs = [Input(shape = (att_lstm_seq_len, feature_vec_len,), name = "att_lstm_input_{0}".format(att+1)) for att in range(att_lstm_num)]
## Create outputs
model = Model(inputs=x, outputs=[out1, out2, out3])
print("Creating model")
def hybrid_controller_model(self, controller_input_shape,
controller_batch_size):
"""
Generate an hybrid LSTM controller with accuracy predictor.
"""
if controller_batch_size > 0:
main_input = Input(shape=controller_input_shape, name='main_input')
else:
main_input = Input(batch_shape=(controller_batch_size,
*controller_input_shape),
name='main_input')
# LSTM layer that processes input
X = LSTM(self.controller_lstm_dim, return_sequences=True)(main_input)
# Predictor single dense layer with sigmoid from LSTM's output
predictor_output = Dense(1,
activation='sigmoid',
name='predictor_output')(X)
# Output single dense layer with softmax from LSTM's output
main_output = Dense(self.controller_classes,
activation='softmax',
name='main_output')(X)
# Return the model with multiple outputs
return Model(inputs=[main_input],
outputs=[main_output, predictor_output])
def test_include_layers_with_nested_input(self):
class PLMergeNest(PLMerge):
def call(self, inputs):
a = inputs[0]
b = inputs[1][0]
c = inputs[1][1]
return a + b + c
x0 = Input(shape=(3, ))
x1 = Input(shape=(3, ))
x2 = Input(shape=(3, ))
l0 = PLMergeNest()
y = l0([x0, [x1, x2]])
stage = preprocessing_stage.FunctionalPreprocessingStage([x0, x1, x2],
y)
stage.compile()
one_array = np.ones((4, 3), dtype='float32')
adapt_data = tf.data.Dataset.from_tensor_slices((one_array, ) * 3)
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 1)
# Check call
y = stage([
tf.ones((4, 3), dtype='float32'),
tf.ones((4, 3), dtype='float32'),
tf.ones((4, 3), dtype='float32')
])
self.assertAllClose(y, np.ones((4, 3), dtype='float32') + 2.)
def test_include_layers_with_dict_input(self):
class PLMergeDict(PLMerge):
def call(self, inputs):
return inputs['a'] + inputs['b']
x0 = Input(shape=(3, ))
x1 = Input(shape=(3, ))
l0 = PLMergeDict()
y = l0({'a': x0, 'b': x1})
l1 = PLSplit()
z, y = l1(y)
stage = preprocessing_stage.FunctionalPreprocessingStage([x0, x1],
[y, z])
stage.compile()
one_array = np.ones((4, 3), dtype='float32')
adapt_data = tf.data.Dataset.from_tensor_slices((one_array, one_array))
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
# Check call
y, z = stage([
tf.ones((4, 3), dtype='float32'),
tf.ones((4, 3), dtype='float32')
])
self.assertAllClose(y, np.ones((4, 3), dtype='float32'))
self.assertAllClose(z, np.ones((4, 3), dtype='float32') + 2.)
def build_net(CATES=12,
height=64,
width=64,
channel=3,
using_white_norm=True,
using_SE=True):
base_model = build_shared_plain_network(using_white_norm=using_white_norm,
using_SE=using_SE)
print(base_model.summary())
x1 = Input(shape=(height, width, channel))
x2 = Input(shape=(height, width, channel))
x3 = Input(shape=(height, width, channel))
y1 = base_model(x1)
y2 = base_model(x2)
y3 = base_model(x3)
cfeat = Concatenate(axis=-1)([y1, y2, y3])
bulk_feat = Dense(CATES,
use_bias=True,
activity_regularizer=regularizers.l1(0),
activation=softmax)(cfeat)
age = Dense(1, name="age")(bulk_feat)
#age = Lambda(lambda a: tf.reshape(tf.reduce_sum(a * tf.constant([[x * 10.0 for x in xrange(12)]]), axis=-1), shape=(-1, 1)), name="age")(bulk_feat)
return Model(input=[x1, x2, x3], output=[age, bulk_feat])
def build_model(char_num=52, max_char_per_word=30):
"""
:param sequence_len: sequence length
:param char_num: how many char from a to z, A to Z
:return:
"""
word_input = Input((
None,
300,
))
char_input = Input((
None,
max_char_per_word,
))
# x = (batch_size, sequence_len, max_char_per_word)
x = TimeDistributed(
Embedding(input_dim=char_num,
output_dim=100,
input_length=max_char_per_word))(char_input)
# x = (batch_size, sequence_len, max_char_per_word, output_dim)
print("---x shape---")
print(x.shape)
x = TimeDistributed(Conv1D(filters=300, kernel_size=3))(x)
# x = (batch_size, sequence_len, new_steps, filters)
## new_steps = sequence_len-kernel_size+1
## filters = num of filters
print("---x shape---")
print(x.shape)
x = TimeDistributed(MaxPooling1D(pool_size=max_char_per_word - 3 + 1))(x)
# x = (batch_size, sequence_len, downsampled_steps, features)
print("---x shape---")
print(x.shape)
x = Lambda(lambda x: K.squeeze(x, axis=2))(x)
# x = (batch_size, sequence_len, features)
print("---x shape---")
print(x.shape)
x = Concatenate(axis=2)([x, word_input])
# x = (batch_size, sequence_len, 600)
x = Bidirectional(GRU(units=300))(x)
# x = (batch_size, sequence_len, 300+300)
output = Dense(units=11, activation='softmax')(x)
# x = (batch_size, sequence_len, 11)
model = Model(inputs=[char_input, word_input], outputs=output)
model.compile(optimizer='adam', loss='categorical_crossentropy')
model.summary()
return model
def show_ask_attend_answer(vocab_size, num_glimpses=2, n=14):
# Define network inputs where n is the feature rows and columns. In the
# paper they use ResNet 152 res5c features with size (14x14x2048)
image_input = Input(shape=(n, n, 2048))
question_input = Input(shape=(15, ))
# Learn word embeddings in relation to total vocabulary
question_embedding = Embedding(vocab_size, 300,
input_length=15)(question_input)
question_embedding = Activation('tanh')(question_embedding)
question_embedding = Dropout(0.5)(question_embedding)
# LSTM to seuqentially embed word vectors into a single question vector
question_lstm = LSTM(1024)(question_embedding)
# Repeating and tiling question vector to match image input for concatenation
question_tile = RepeatVector(n * n)(question_lstm)
question_tile = Reshape((n, n, 1024))(question_tile)
# Concatenation of question vector and image features
concatenated_features1 = concatenate([image_input, question_tile])
concatenated_features1 = Dropout(0.5)(concatenated_features1)
# Stacked attention network
attention_conv1 = Conv2D(512, (1, 1))(concatenated_features1)
attention_relu = Activation('relu')(attention_conv1)
attention_relu = Dropout(0.5)(attention_relu)
attention_conv2 = Conv2D(num_glimpses, (1, 1))(attention_relu)
attention_maps = Activation('softmax')(attention_conv2)
# Weighted average of image features using attention maps
image_attention = glimpse(attention_maps, image_input, num_glimpses, n)
# Concatenation of question vector and attended image features
concatenated_features2 = concatenate([image_attention, question_lstm])
concatenated_features2 = Dropout(0.5)(concatenated_features2)
# First fully connected layer with relu and dropout
fc1 = Dense(1024)(concatenated_features2)
fc1_relu = Activation('relu')(fc1)
fc1_relu = Dropout(0.5)(fc1_relu)
# Final fully connected layer with softmax to output answer probabilities
fc2 = Dense(3000)(fc1_relu)
fc2_softmax = Activation('softmax')(fc2)
# Instantiate the model
vqa_model = Model(inputs=[image_input, question_input],
outputs=fc2_softmax)
return vqa_model
def test_preprocessing_stage_with_nested_input(self):
# Test with NumPy array
x0 = Input(shape=(3, ))
x1 = Input(shape=(3, ))
x2 = Input(shape=(3, ))
l0 = PLMerge()
y = l0([x0, x1])
l1 = PLMerge()
y = l1([y, x2])
l2 = PLSplit()
z, y = l2(y)
stage = preprocessing_stage.FunctionalPreprocessingStage(
[x0, [x1, x2]], [y, z])
stage.compile()
one_array = np.ones((4, 3), dtype='float32')
stage.adapt([one_array, [one_array, one_array]])
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertEqual(l2.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Check call
y, z = stage([
tf.ones((4, 3), dtype='float32'),
[
tf.ones((4, 3), dtype='float32'),
tf.ones((4, 3), dtype='float32')
]
])
self.assertAllClose(y, np.ones((4, 3), dtype='float32') + 1.)
self.assertAllClose(z, np.ones((4, 3), dtype='float32') + 3.)
# Test with dataset
adapt_data = tf.data.Dataset.from_tensor_slices(
(one_array, (one_array, one_array)))
adapt_data = adapt_data.batch(2) # 5 batches of 2 samples
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 2)
self.assertEqual(l1.adapt_count, 2)
self.assertEqual(l2.adapt_count, 2)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
self.assertLessEqual(l1.adapt_time, l2.adapt_time)
# Test error with bad data
with self.assertRaisesRegex(ValueError, 'requires a '):
stage.adapt(None)
def _setup_code2vec_input_layer(self, input_params):
encoder_name = input_params["encoder"]
encoder_params = self.args["encoders"][encoder_name]
max_seq_len = encoder_params["max_seq_len"]
start_token_input = Input(
shape=(max_seq_len, ),
name="ast_path_start",
)
path_input = Input(
shape=(max_seq_len, ),
name="ast_path_index",
)
end_token_input = Input(
shape=(max_seq_len, ),
name="ast_path_end",
)
self.inputs += [start_token_input, path_input, end_token_input]
token_embedding_shared_layers = Embedding(
input_dim=encoder_params["token_vocab_size"],
output_dim=encoder_params["token_emb_size"],
name="token_embedding",
mask_zero=True,
input_length=max_seq_len)
path_embedding = Embedding(input_dim=encoder_params["path_vocab_size"],
output_dim=encoder_params["path_emb_size"],
name="path_embedding",
mask_zero=True)
path_start_emb = token_embedding_shared_layers(start_token_input)
path_end_emb = token_embedding_shared_layers(end_token_input)
path_emb = path_embedding(path_input)
context_emb = Concatenate(name="ast_context")(
[path_start_emb, path_emb, path_end_emb])
context_emb = Dropout(encoder_params["dropout"])(context_emb)
context_dense = Dense(2 * encoder_params["token_emb_size"] +
encoder_params["path_emb_size"],
use_bias=False,
activation="tanh")
context_after_dense = TimeDistributed(context_dense)(context_emb)
layer = self._setup_attention(encoder_params, context_after_dense,
encoder_name)
return layer
def generate_model(title, selftext, link):
"""Generate an ML model based on the content type of the subreddit"""
inputs, models = [], []
#title model
if title:
t_in = Input(shape=(2048, ))
t_embed = Embedding(2048, 16)(t_in)
t_dense1 = Dense(16, activation="relu")(t_embed)
t_dense2 = Dense(16, activation="relu")(t_dense1)
t_dense3 = Dense(16, activation="relu")(t_dense2)
t_dense4 = Dense(1, activation="relu")(t_dense3)
t_flat = Flatten()(t_dense4)
inputs.append(t_in)
models.append(t_flat)
#selftext model
if selftext:
s_in = Input(shape=(2048, ))
s_embed = Embedding(2048, 16)(s_in)
s_dense1 = Dense(16, activation="relu")(s_embed)
s_dense2 = Dense(16, activation="relu")(s_dense1)
s_dense3 = Dense(16, activation="relu")(s_dense2)
s_dense4 = Dense(1, activation="relu")(s_dense3)
s_flat = Flatten()(s_dense4)
inputs.append(s_in)
models.append(s_flat)
#link model
if link:
l_in = Input(shape=(128, 128, 3))
l_conv1 = Conv2D(32, (3, 3),
input_shape=(3, 128, 128),
activation="relu")(l_in)
l_pool1 = MaxPooling2D(pool_size=(2, 2))(l_conv1)
l_conv2 = Conv2D(32, (3, 3), activation="relu")(l_pool1)
l_pool2 = MaxPooling2D(pool_size=(2, 2))(l_conv2)
l_flat1 = Flatten()(l_pool2)
l_dense1 = Dense(16, activation="relu")(l_flat1)
l_dense2 = Dense(1, activation="relu")(l_dense1)
inputs.append(l_in)
models.append(l_dense2)
if len(models) > 1:
merge = Concatenate()(models)
else:
merge = models[0]
m_dense1 = Dense(16, activation="relu")(merge)
m_out = Dense(2, activation="relu")(m_dense1)
model = Model(inputs=inputs, outputs=m_out)
print(model.summary())
model.compile(loss='mse', optimizer='adam', metrics=['mse'])
return model
def make_configuration():
input_shape = (sequence_length,)
model_input = Input(shape=input_shape)
z = BatchNormalization()(model_input)
z = Embedding(len(vocabulary_inv),
embedding_dim,
input_length=sequence_length,
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = BatchNormalization()(conv)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = BatchNormalization()(z)
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
z = BatchNormalization()(z)
model_output = Dense(N, activation="softmax")(z)
model = Model(model_input, model_output)
return model
def build_model(X_train, X_test, Y_train, noLSTM, train_labels):
model = Sequential()
model.reset_states()
# with codecs.open(rootFolder + "training.csv", 'a') as logfile:
# fieldnames = ['lstm1', 'lstm2', 'dense1', 'dense2', 'dense3']
# writer = csv.DictWriter(logfile, fieldnames=fieldnames)
# writer.writerow({'lstm1': noLSTM[0], 'lstm2': noLSTM[1],
# 'dense1': noLSTM[2], 'dense2': noLSTM[3] , 'dense3': noLSTM[4]})
# input
dropoutRate = 0.5
Chans = X_train.shape[2]
Samples = slidingWindowSize
input_main = Input(X_train.shape)
block1 = Conv2D(40, (1, 13),
input_shape=(X_train.shape),
kernel_constraint=max_norm(2.))(input_main)
block1 = Conv2D(40, (Chans, 1), use_bias=False,
kernel_constraint=max_norm(2.))(block1)
block1 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block1)
block1 = Activation(square)(block1)
block1 = AveragePooling2D(pool_size=(1, 35), strides=(1, 7))(block1)
block1 = Activation(log)(block1)
block1 = Dropout(dropoutRate)(block1)
flatten = Flatten()(block1)
dense = Dense(3, kernel_constraint=max_norm(0.5))(flatten)
softmax = Activation('softmax')(dense)
# ['acc', 'loss', 'val_acc', 'val_loss']
opt = Adam(lr=0.0011, decay=0.001)
model.compile(optimizer=opt, loss='categorical_crossentropy', metrics=['accuracy'])
fnametmp = rootFolder + "plot_{}_{}_{}_{}_{}.png".format("Model", noLSTM[0], noLSTM[1], noLSTM[2], noLSTM[3], noLSTM[4])
plot_model(model, to_file=fnametmp, show_shapes=True,
show_layer_names=True, rankdir='TB')
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0,
patience=3, verbose=1, mode='auto')
tn = TerminateOnNaN()
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.2, min_lr=1e-7, verbose=2)
checkpoint_path = os.path.join(rootFolder, "weights.best_{}_{}_{}_{}_{}.hdf5".format("model", noLSTM[0], noLSTM[1], noLSTM[2], noLSTM[3], noLSTM[4]))
checkpoint = ModelCheckpoint(checkpoint_path, monitor='val_acc', verbose=1,
save_best_only=True, mode='max')
csv_logger = CSVLogger(rootFolder + 'training.csv', append=True)
early_stop = EarlyStopping(monitor='val_acc', patience=1, verbose=2, mode='auto')
callback_fns = [early_stopping, tn, csv_logger, checkpoint, reduce_lr]
history = model.fit(X_train, Y_train, batch_size=20, epochs=5,
callbacks=callback_fns, validation_split=0.3, shuffle=True)
fnametmp = rootFolder + "model_{}_{}_{}_{}_{}".format(noLSTM[0], noLSTM[1], noLSTM[2], noLSTM[3], noLSTM[4])
model.save_weights(fnametmp + '.h5')
with open(fnametmp + '.json', 'w') as f:
f.write(model.to_json())
fnametmp = "plot-{}-{}-{}.png".format("model-accuracy", noLSTM[0], noLSTM[1])
drawMe(yVal=history.history['acc'], xVal=history.history['val_acc'],
title='model accuracy', xlabel='epoch', ylabel='accuracy', legend=['train', 'test'], save=True,
fileName=fnametmp, show=False)
fnametmp = "plot-{}-{}-{}.png".format("model-loss", noLSTM[0], noLSTM[1])
drawMe(yVal=history.history['loss'], xVal=history.history['val_loss'],
title='model loss', xlabel='epoch', ylabel='loss', legend=['train', 'test'], save=True,
fileName=fnametmp, show=False)
def ResNet(input_shape=(224, 224, 3), layer_size=152, output_size=1000):
# If you would like to try another layer/block sizes, add it in this dictionary and change the layer_size parameter when initializing ResNet.
blocks_for_stages = {18: [2, 2, 2, 2],
34: [3, 4, 6, 3],
50: [3, 4, 6, 3],
101: [3, 4, 23, 3],
152: [3, 8, 36, 3]}
x_input = Input(input_shape)
x = first_stage(x_input)
bottleneck = layer_size in [101, 152]
filters = 32
for stage in range(len(blocks_for_stages[layer_size])):
filters *= 2
for block in range(blocks_for_stages[layer_size][stage]):
if block is 0:
x = residual_block(x, filters, first_layer=True, bottleneck=bottleneck)
else:
x = residual_block(x, filters, bottleneck=bottleneck)
x = output_stage(x, output_size)
model = Model(inputs = x_input, outputs = x, name='ResNet')
return model
def chargen_model(num_of_classes, cfg, weights_filepath=None, dropout=0.3, optimizer=Adam(decay=2**-20)):
"""Builds the neural network model architecture for CharGen and loads the weights specified for the model.
:param num_of_classes: total number of classes
:param cfg: configuration of CharGen
:param weights_filepath: path of the weights file
:param dropout: fraction of neurons to be ignored in a single forward and backward pass (default 0.05)
:param optimizer: specify which optimization algorithm to use (Default RMSprop)
:return: model to be used for training
"""
inp = Input(shape=(cfg['input_length'],), name='input')
embedded = Embedding(num_of_classes, cfg['embedding_dims'],
input_length=cfg['input_length'],
name='embedding')(inp)
if dropout > 0.0:
embedded = SpatialDropout1D(dropout, name='dropout')(embedded)
rnn_layer_list = []
for i in range(cfg['rnn_layers']):
prev_layer = embedded if i is 0 else rnn_layer_list[-1]
rnn_layer_list.append(new_rnn_layer(cfg, i + 1)(prev_layer))
seq_concat = concatenate([embedded] + rnn_layer_list, name='rnn_concat')
attention = WeightedAttentionAverage(name='attention')(seq_concat)
output = Dense(num_of_classes, name='output', activation='softmax')(attention)
model = Model(inputs=[inp], outputs=[output])
if weights_filepath is not None:
model.load_weights(weights_filepath, by_name=True)
model.compile(loss='categorical_crossentropy', optimizer=optimizer)
return model
def autoencoder_Dropout(params, dropout=0.2, encodingDim=3):
X = params['X_train']
name = params['name']
args = Args(params['args'])
if encodingDim > 3:
encodingDim = 3
input_d = Input(shape=(X.shape[1], ))
encoded = Dense(6, activation='tanh')(input_d)
encoded = Dropout(dropout)(encoded)
encoded = Dense(5, activation='tanh')(encoded)
encoded = Dropout(dropout)(encoded)
encoded = Dense(4, activation='tanh')(encoded)
encoded = Dropout(dropout)(encoded)
encoded = Dense(encodingDim, activation='tanh')(encoded)
#encoded = Dropout(dropout)(encoded)
decoded = Dense(4, activation='tanh')(encoded)
#decoded = Dropout(dropout)(decoded)
decoded = Dense(5, activation='tanh')(decoded)
#decoded = Dropout(dropout)(decoded)
decoded = Dense(6, activation='tanh')(decoded)
#decoded = Dropout(dropout)(decoded)
decoded = Dense(X.shape[1], activation='linear')(decoded)
model = Model(input_d, decoded)
return AutoencoderModel(model, X, args, modelType="AUTOENCODER", name=name)
def style_considered_model(inputs_shape, num_classes=None):
"""
concerning style information
"""
input_shape, sbow_shape = inputs_shape
model = keras.applications.DenseNet169(input_shape=input_shape,
include_top=False)
# frozen
model.trainable = False
x1_1 = GlobalAveragePooling2D()(model.layers[-1].output)
x1_2 = GlobalAveragePooling2D()(model.layers[-9].output)
x1_3 = GlobalAveragePooling2D()(model.layers[-23].output)
x2_1 = Dense(512, activation='elu')(x1_1)
x2_2 = Dense(512, activation='elu')(x1_2)
x2_3 = Dense(512, activation='elu')(x1_3)
sbow = Input(shape=sbow_shape, name="style_bow")
con = concatenate([x2_1, x2_2, x2_3, sbow], axis=-1)
sdes = Dense(512, kernel_regularizer='l2')(con)
cos = CosineTheta(num_classes, 512)(sdes)
model_new = Model(inputs=[model.input, sbow], outputs=cos)
return model_new
def test_adapt_preprocessing_stage_with_dict_output(self):
x = Input(shape=(3, ), name='x')
l0 = PLSplit()
y0, y1 = l0(x)
l1 = PLSplit()
z0, z1 = l1(y0)
stage = preprocessing_stage.FunctionalPreprocessingStage({'x': x}, {
'y1': y1,
'z1': z1,
'y0': y0,
'z0': z0
})
stage.compile()
# Test with NumPy array
one_array = np.ones((4, 3), dtype='float32')
adapt_data = {'x': one_array}
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
# Check call
outputs = stage({'x': tf.constant(one_array)})
self.assertEqual(set(outputs.keys()), {'y0', 'y1', 'z0', 'z1'})
self.assertAllClose(outputs['y0'],
np.ones((4, 3), dtype='float32') + 1.)
self.assertAllClose(outputs['y1'],
np.ones((4, 3), dtype='float32') - 1.)
self.assertAllClose(outputs['z0'],
np.ones((4, 3), dtype='float32') + 2.)
self.assertAllClose(outputs['z1'], np.ones((4, 3), dtype='float32'))
def create_model(config):
indata = Input((config.window_len, config.token_dims))
# split into the hash and the rest of the token features, embed hash as one-hot, then merge
tok_hash = Lambda(lambda x: squeeze(
K.backend.slice(x, (0, 0, 0), (-1, -1, 1)), axis=2))(indata)
tok_features = Lambda(lambda x: K.backend.slice(x, (0, 0, 1),
(-1, -1, -1)))(indata)
embed = Embedding(config.vocab_size, 32)(tok_hash)
merged = concatenate([embed, tok_features], axis=2)
f = Flatten()(merged)
d1 = Dense(config.window_len * config.token_dims * 5,
activation='sigmoid')(f)
d2 = Dropout(0.3)(d1)
d3 = Dense(config.window_len * config.token_dims, activation='sigmoid')(d2)
d4 = Dropout(0.3)(d3)
d5 = Dense(config.window_len, activation='elu')(d4)
model = Model(inputs=[indata], outputs=[d5])
model.compile(optimizer='adam',
loss=missed_token_loss(config.penalize_missed),
metrics=['acc'])
return model
def test_adapt_preprocessing_stage_with_dict_output(self):
x = Input(shape=(3, ), name="x")
l0 = PLSplit()
y0, y1 = l0(x)
l1 = PLSplit()
z0, z1 = l1(y0)
stage = preprocessing_stage.FunctionalPreprocessingStage({"x": x}, {
"y1": y1,
"z1": z1,
"y0": y0,
"z0": z0
})
stage.compile()
# Test with NumPy array
one_array = np.ones((4, 3), dtype="float32")
adapt_data = {"x": one_array}
stage.adapt(adapt_data)
self.assertEqual(l0.adapt_count, 1)
self.assertEqual(l1.adapt_count, 1)
self.assertLessEqual(l0.adapt_time, l1.adapt_time)
# Check call
outputs = stage({"x": tf.constant(one_array)})
self.assertEqual(set(outputs.keys()), {"y0", "y1", "z0", "z1"})
self.assertAllClose(outputs["y0"],
np.ones((4, 3), dtype="float32") + 1.0)
self.assertAllClose(outputs["y1"],
np.ones((4, 3), dtype="float32") - 1.0)
self.assertAllClose(outputs["z0"],
np.ones((4, 3), dtype="float32") + 2.0)
self.assertAllClose(outputs["z1"], np.ones((4, 3), dtype="float32"))
def _build_model(self):
input_layer = Input(shape=(self.state_size, ))
x = Dense(64, activation="relu")(input_layer)
x = Dense(64, activation="relu")(x)
x = Dense(self.action_size, activation="linear")(x)
model = Model(inputs=input_layer, outputs=x)
model.compile(loss="mean_squared_error", optimizer="Adam")
return model
def kerasLSTM(
params,
layers=[128],
dropout=0.0,
recurrentDropout=0.0,
alpha=None,
training=False,
):
X_train = params['X_train']
y_train = params['y_train']
name = params['name']
args = Args(params['args'])
input_layer = Input(shape=(None, X_train.shape[-1]))
if len(layers) > 1:
firstLayerUnits = layers[0]
layer_1 = LSTM(firstLayerUnits,
activation=args.activation,
dropout=dropout,
recurrent_dropout=recurrentDropout,
return_sequences=True)(input_layer, training=training)
if alpha is not None:
layer_1 = LeakyReLU(alpha=alpha)(layer_1)
for i, layerUnits in enumerate(layers[1:]):
layer_1 = LSTM(layerUnits,
activation=args.activation,
dropout=dropout,
recurrent_dropout=recurrentDropout,
return_sequences=True if
(i < len(layers) - 2) else False)(layer_1,
training=training)
if alpha is not None:
layer_1 = LeakyReLU(alpha=alpha)(layer_1)
else:
firstLayerUnits = layers[0]
layer_1 = LSTM(firstLayerUnits,
activation=args.activation,
dropout=dropout,
return_sequences=False,
recurrent_dropout=recurrentDropout)(input_layer,
training=training)
if alpha is not None:
layer_1 = LeakyReLU(alpha=alpha)(layer_1)
output_layer = Dense(y_train.shape[-1], activation='linear')(layer_1)
model = Model(input_layer, output_layer)
return MachinLearningModel(
model,
X_train,
y_train,
args=args,
modelType="RNN",
name=name,
)
def ResNet18(input_shape=None, classes=10, **kwargs):
# Define the input as a tensor with shape input_shape
x_input = Input(input_shape)
if backend.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
#stage 1
with tf.name_scope('stage1'):
x = layers.ZeroPadding2D(padding=(3, 3), name='conv1_pad')(x_input)
x = layers.Conv2D(64, (7, 7),
strides=(2, 2),
padding='valid',
kernel_initializer='he_normal',
name='conv1')(x)
x = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = layers.Activation('relu')(x)
print("stage1:" + str(x.shape))
#stage 2
with tf.name_scope('stage2'):
x = layers.ZeroPadding2D(padding=(1, 1), name='pool1_pad')(x)
x = layers.MaxPooling2D((3, 3), strides=(2, 2))(x)
x = identity_block(x, 3, [64, 64], stage=2, block='b')
x = identity_block(x, 3, [64, 64], stage=2, block='c')
print("stage2:" + str(x.shape))
#stage 3
with tf.name_scope('stage3'):
x = conv_block(x, 3, [128, 128], stage=3, block='a')
x = identity_block(x, 3, [128, 128], stage=3, block='d')
print("stage3:" + str(x.shape))
#stage 4
with tf.name_scope('stage4'):
x = conv_block(x, 3, [256, 256], stage=4, block='a')
x = identity_block(x, 3, [256, 256], stage=4, block='c')
print("stage4:" + str(x.shape))
#stage 5
with tf.name_scope('stage5'):
x = conv_block(x, 3, [512, 512], stage=5, block='a')
x = identity_block(x, 3, [512, 512], stage=5, block='c')
print("stage5:" + str(x.shape))
#full-connected layer
with tf.name_scope('fc'):
x = layers.GlobalAveragePooling2D(name='avg_pool')(x)
x = layers.Dense(classes, activation='softmax', name='fc10')(x)
# Create model.
model = models.Model(x_input, x, name='resnet18')
return model
def _build_model(self):
input_layer = Input(shape=(self.img_height, self.img_width, 4))
x = Conv2D(16, (8, 8), strides=4, activation="relu")(input_layer)
x = Conv2D(32, (4, 4), strides=2, activation="relu")(x)
x = Dense(256, activation="relu")(x)
x = Dense(self.action_size, activation="linear")(x)
model = Model(inputs=input_layer, outputs=x)
model.compile(loss="mean_squared_error",
optimizer=Adam(lr=self.learning_rate))
return model
def control_model(self, controller_input_shape, controller_batch_size):
main_input = Input(shape=controller_input_shape,
batch_shape=controller_batch_size,
name='main_input')
x = LSTM(self.controller_lstm_dim, return_sequences=True)(main_input)
main_output = Dense(self.controller_classes,
activation='softmax',
name='main_output')(x)
model = Model(inputs=[main_input], outputs=[main_output])
return model
def _build_model(self):
input_layer = Input(shape=(2, ))
x = Dense(64, activation="relu")(input_layer)
x = Dense(64, activation="relu")(x)
x = Dense(self.action_size, activation="linear")(x)
model = Model(inputs=input_layer, outputs=x)
model.compile(loss="mean_squared_error",
optimizer=keras.optimizers.Adam(lr=0.001))
model.summary()
return model
def create(self, res: Resolution, classes: Optional[int]) -> Model:
"""
Creates a keras.models.Model object.
"""
if type(classes) is not int:
raise ValueError(classes)
return VGG16(
include_top=True,
weights=None,
input_tensor=Input(res.hwc()),
classes=classes,
)
def RBF_model(self, x, input_shape, units, betas, loss):
inputs = Input(shape=(input_shape, ))
rbflayer = RBFLayer(output_dim=units,
betas=betas,
initializer=InitCentersRandom(x))
rbf = rbflayer(inputs)
out = Dense(1)(rbf)
model = Model(inputs=inputs, outputs=out)
model.compile(loss=loss, optimizer=RMSprop())
return model
def MLP_model(self, input_shape, loss):
inputs = Input(shape=(input_shape, ))
layer = Dense(128, activation=K.sigmoid)(inputs)
lay = Dense(64, activation=K.sigmoid)(layer)
out = Dense(1)(lay)
model = Model(inputs=inputs, outputs=out)
model.compile(loss=loss, optimizer=RMSprop())
return model