def _build_model_small(n_classes=5, activation='elu', dropout_1_rate=0.25, dropout_2_rate=0.25,
reg_factor=200e-4, bias_reg_factor=None, batch_norm=False):
l2_reg = regularizers.l2(reg_factor)
l2_bias_reg = None
if bias_reg_factor:
regularizers.l2(bias_reg_factor)
# input image dimensions
h, w, d = 32, 32, 3
if K.image_data_format() == 'channels_first':
input_shape = (3, h, w)
else:
input_shape = (h, w, 3)
# input image dimensions
x = input_1 = Input(shape=input_shape)
x = Conv2D(filters=4, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(
x)
# if batch_norm:
# x = BatchNormalization()(x)
# x = Activation(activation=activation)(x)
# x = Conv2D(filters=4, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)
# if batch_norm:
# x = BatchNormalization()(x)
x = Activation(activation=activation)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(rate=dropout_1_rate)(x)
x = Conv2D(filters=8, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(
x)
# if batch_norm:
# x = BatchNormalization()(x)
# x = Activation(activation=activation)(x)
# x = Conv2D(filters=8, kernel_size=(3, 3), padding='same', kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)
# if batch_norm:
# x = BatchNormalization()(x)
x = Activation(activation=activation)(x)
x = MaxPooling2D(pool_size=(2, 2))(x)
x = Dropout(rate=dropout_1_rate)(x)
x = Flatten()(x)
x = Dense(units=8, kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)
if batch_norm:
x = BatchNormalization()(x)
x = Activation(activation=activation)(x)
x = Dropout(rate=dropout_2_rate)(x)
x = Dense(units=n_classes, kernel_regularizer=l2_reg, bias_regularizer=l2_bias_reg)(x)
if batch_norm:
x = BatchNormalization()(x)
x = Activation(activation='softmax')(x)
return Model(inputs=[input_1], outputs=[x])
def get_unet():
inputs = Input((img_rows, img_cols, depth))
up1 = block(inputs, 16, 32, True)
up2 = block(up1, 32, 64, True)
up3 = block(up2, 64, 128, True)
up4 = block(up3, 128, 256, True)
up5 = block(up4, 256, 512, True)
# **** decoding ****
xx = concatenate([
Conv2DTranspose(256, (2, 2), strides=(2, 2), padding='same')(up5), up4
],
axis=3)
down1 = block(xx, 512, 128, False)
xx = concatenate([
Conv2DTranspose(256,
(2, 2), strides=(2, 2), padding='same')(down1), up3
],
axis=3)
down2 = block(xx, 256, 64, False)
xx = concatenate([
Conv2DTranspose(128,
(2, 2), strides=(2, 2), padding='same')(down2), up2
],
axis=3)
down3 = block(xx, 128, 32, False)
xx = concatenate([
Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(down3), up1
],
axis=3)
down4 = block(xx, 64, 16, False)
xx = concatenate([
Conv2DTranspose(16,
(2, 2), strides=(2, 2), padding='same')(down4), inputs
],
axis=3)
xx = Conv2D(32, (3, 3), activation='relu', padding='same')(xx)
# xx = concatenate([xx, conv1a])
xx = Conv2D(1, (1, 1), activation='sigmoid', padding='same')(xx)
model = Model(inputs=[inputs], outputs=[xx])
return model
def train_f_enc(self, steps_list, epoch=50):
# TODO what's add0 and add1?
print("training f_enc")
f_add0 = Sequential(name='f_add0')
f_add0.add(self.f_enc)
f_add0.add(Dense(FIELD_DEPTH))
f_add0.add(Activation('softmax', name='softmax_add0'))
f_add1 = Sequential(name='f_add1')
f_add1.add(self.f_enc)
f_add1.add(Dense(FIELD_DEPTH))
f_add1.add(Activation('softmax', name='softmax_add1'))
# f_mul0 = Sequential(name='f_mul0')
# f_mul0.add(self.f_enc)
# f_mul0.add(Dense(FIELD_DEPTH))
# f_mul0.add(Activation('softmax', name='softmax_mul0'))
#
# f_mul1 = Sequential(name='f_mul1')
# f_mul1.add(self.f_enc)
# f_mul1.add(Dense(FIELD_DEPTH))
# f_mul1.add(Activation('softmax', name='softmax_mul1'))
env_model = Model(self.f_enc.inputs, [f_add0.output, f_add1.output], name="env_model")
env_model.compile(optimizer='adam', loss=['categorical_crossentropy']*2)
# TODO adjsut the model
for ep in range(epoch):
losses = []
for idx, steps_dict in enumerate(steps_list):
prev = None
for step in steps_dict['steps']:
# step.input = env + program + argument
x = self.convert_input(step.input)[:2]
env_values = step.input.env.reshape((FIELD_ROW, -1))
# ??
in1 = np.clip(env_values[0].argmax() - 1, 0, 9)
in2 = np.clip(env_values[1].argmax() - 1, 0, 9)
carry = np.clip(env_values[2].argmax() - 1, 0, 9)
y_num = in1 + in2 + carry
# mul1 = np.clip(env_values[4].argmax() - 1, 0, 9)
# mul2 = np.clip(env_values[5].argmax() - 1, 0, 9)
now = (in1, in2, carry)
## no need for training? why?
if prev == now:
continue
prev = now
y0 = to_one_hot_array((y_num % 10)+1, FIELD_DEPTH)
y1 = to_one_hot_array((y_num // 10)+1, FIELD_DEPTH)
y = [yy.reshape((self.batch_size, -1)) for yy in [y0, y1]]
loss = env_model.train_on_batch(x, y)
losses.append(loss)
print("in train_f_enc: ep %3d: loss=%s" % (ep, np.average(losses)))
if np.average(losses) < 1e-06:
break
def create_policy_value_net(self):
"""create the policy value network """
in_x = network = Input((4, self.board_width, self.board_height))
# conv layers
network = Conv2D(filters=32,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(network)
network = Conv2D(filters=64,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(network)
network = Conv2D(filters=128,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(network)
# action policy layers
policy_net = Conv2D(filters=4,
kernel_size=(1, 1),
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(network)
policy_net = Flatten()(policy_net)
self.policy_net = Dense(self.board_width * self.board_height,
activation="softmax",
kernel_regularizer=l2(
self.l2_const))(policy_net)
# state value layers
value_net = Conv2D(filters=2,
kernel_size=(1, 1),
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(network)
value_net = Flatten()(value_net)
value_net = Dense(64, kernel_regularizer=l2(self.l2_const))(value_net)
self.value_net = Dense(1,
activation="tanh",
kernel_regularizer=l2(self.l2_const))(value_net)
self.model = Model(in_x, [self.policy_net, self.value_net])
def policy_value(state_input):
state_input_union = np.array(state_input)
results = self.model.predict_on_batch(state_input_union)
return results
self.policy_value = policy_value
def create_desnet121():
base = applications.DenseNet121(include_top=False,
weights=None,
input_shape=(112, 112, 3),
pooling='max')
x = base.output
x = Dropout(0.2)(x)
output = Dense(1, activation='linear')(x)
return Model(inputs=base.input, outputs=output)
def build(self):
dim_data = self.size_of_input_data_dim
nb_time_step = self.size_of_input_timesteps
news_input = Input(shape=(nb_time_step, dim_data), name='x1')
lstm = LSTM(output_dim=nb_hidden_units, dropout_U=dropout, dropout_W=dropout,
W_regularizer=l2(l2_norm_alpha), b_regularizer=l2(l2_norm_alpha),
activation='tanh', name='h1')
bi_lstm = Bidirectional(lstm, input_shape=(nb_time_step, dim_data), merge_mode='concat', name='h1')
all_news_rep = bi_lstm(news_input)
news_predictions = Dense(1, activation='linear')(all_news_rep)
self.model = Model(news_input, news_predictions, name="deep rnn for financial news analysis")
def _build_single_device_model(blueprint, device):
import tensorflow as tf
with tf.device(get_logical_device(device)):
inputs = Input(shape=(blueprint.layout.input_size, ))
row_input = inputs
for row in blueprint.layout.rows:
row_input = _build_row_model(row_input, row)
final_layer_input = _maybe_merge_inputs(row_input)
predictions = Dense(
blueprint.layout.output_size,
activation=blueprint.layout.output_activation)(final_layer_input)
return Model(input=inputs, output=predictions)
def conv_3d_pre_trained(self):
"""
Build a 3D convolutional network, based loosely on C3D.
https://arxiv.org/pdf/1412.0767.pdf
"""
# Model.
root_dir = './thirdp/c3d_keras/'
model_dir = root_dir + 'models'
model_weight_filename = os.path.join(model_dir,
'sports1M_weights_tf.h5')
model_json_filename = os.path.join(model_dir,
'sports1M_weights_tf.json')
print("[Info] Reading model architecture...")
base_model = model_from_json(open(model_json_filename, 'r').read())
print("[Info] Loading model weights...")
base_model.load_weights(model_weight_filename)
print("[Info] Loading model weights -- DONE!")
for i, layer in enumerate(base_model.layers):
print(i, layer.name)
print(layer.get_output_at(0).get_shape().as_list())
x = base_model.get_layer('pool5').output
x = Flatten()(x)
x = Dense(4096)(x)
x = Dropout(0.2)(x)
x = Dense(4096)(x)
x = Dropout(0.2)(x)
predictions = Dense(self.nb_classes, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
for i, layer in enumerate(model.layers):
print(i, layer.name)
print(layer.get_output_at(0).get_shape().as_list())
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in base_model.layers:
layer.trainable = True
# model.compile(loss='mean_squared_error', optimizer='sgd')
# compile the model (should be done *after* setting layers to non-trainable)
# model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
return model
def build(self):
enc_size = self.size_of_env_observation() #
argument_size = IntegerArguments.size_of_arguments #
input_enc = InputLayer(batch_input_shape=(self.batch_size, enc_size), name='input_enc')
input_arg = InputLayer(batch_input_shape=(self.batch_size, argument_size), name='input_arg')
input_prg = Embedding(input_dim=PROGRAM_VEC_SIZE, output_dim=PROGRAM_KEY_VEC_SIZE, input_length=1,
batch_input_shape=(self.batch_size, 1))
f_enc = Sequential(name='f_enc')
f_enc.add(Merge([input_enc, input_arg], mode='concat'))
f_enc.add(MaxoutDense(128, nb_feature=4))
self.f_enc = f_enc
program_embedding = Sequential(name='program_embedding')
program_embedding.add(input_prg)
f_enc_convert = Sequential(name='f_enc_convert')
f_enc_convert.add(f_enc)
f_enc_convert.add(RepeatVector(1))
f_lstm = Sequential(name='f_lstm')
f_lstm.add(Merge([program_embedding, f_enc_convert], mode='concat'))
f_lstm.add(LSTM(256, return_sequences=False, stateful=True, W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_1'))
f_lstm.add(RepeatVector(1))
f_lstm.add(LSTM(256, return_sequences=False, stateful=True, W_regularizer=l2(0.0000001)))
f_lstm.add(Activation('relu', name='relu_lstm_2'))
f_end = Sequential(name='f_end')
f_end.add(f_lstm)
f_end.add(Dense(1, W_regularizer=l2(0.001)))
f_end.add(Activation('sigmoid', name='sigmoid_end'))
f_prog = Sequential(name='f_prog')
f_prog.add(f_lstm)
f_prog.add(Dense(PROGRAM_KEY_VEC_SIZE, activation="relu"))
f_prog.add(Dense(PROGRAM_VEC_SIZE, W_regularizer=l2(0.0001)))
f_prog.add(Activation('softmax', name='softmax_prog'))
# plot(f_prog, to_file='f_prog.png', show_shapes=True)
f_args = []
for ai in range(1, IntegerArguments.max_arg_num+1):
f_arg = Sequential(name='f_arg%s' % ai)
f_arg.add(f_lstm)
f_arg.add(Dense(IntegerArguments.depth, W_regularizer=l2(0.0001)))
f_arg.add(Activation('softmax', name='softmax_arg%s' % ai))
f_args.append(f_arg)
self.model = Model([input_enc.input, input_arg.input, input_prg.input],
[f_end.output, f_prog.output] + [fa.output for fa in f_args],
name="npi")
self.compile_model()
plot(self.model, to_file='model.png', show_shapes=True)
def test_sparse_input_validation_split():
test_input = sparse.random(6, 3, density=0.25).tocsr()
in1 = Input(shape=(3, ), sparse=True)
out1 = Dense(4)(in1)
test_output = np.random.random((6, 4))
model = Model(in1, out1)
model.compile('rmsprop', 'mse')
model.fit(test_input,
test_output,
epochs=1,
batch_size=2,
validation_split=0.2)
def build(self):
# build the first layer that directly connects to the input
in_layer = x = Input((2, self.config.num_rows, self.config.num_cols))
x = Conv2D(filters=self.config.num_cnn_filters,
kernel_size=self.config.size_cnn_filter,
padding="same",
data_format="channels_first",
kernel_regularizer=l2(self.config.l2_reg))(x)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
# then build the following (hidden) convolutional layers of the network
for layer in range(self.config.num_conv_layers):
x = self._build_conv_block(x)
# now split the network to give separate outputs for the value (i.e. estimated winning probability) and the projected best move on the network-level (i.e. without tree-improvement)
out_intermediate = x
# build the value output chain
x = Conv2D(
filters=1,
kernel_size=1,
data_format="channels_first",
kernel_regularizer=l2(self.config.l2_reg))(
out_intermediate) # cut the dimensionality down to 1 now
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
# add a dense layer as the second-to-last one in the network for the value path
x = Dense(self.config.value_dense_units,
kernel_regularizer=l2(self.config.l2_reg),
activation="relu")(x)
# the last one reduces the dimension down to 1
out_layer_value = Dense(1,
kernel_regularizer=l2(self.config.l2_reg),
activation="tanh",
name="out_layer_value")(x)
# build the move output chain
x = Conv2D(filters=2,
kernel_size=1,
data_format="channels_first",
kernel_regularizer=l2(self.config.l2_reg))(out_intermediate)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
out_layer_move = Dense(self.config.num_cols,
kernel_regularizer=l2(self.config.l2_reg),
activation="softmax",
name="out_layer_move")(x)
self.model = Model(in_layer, [out_layer_move, out_layer_value],
name="c4_model")
def __init__(self,
encoder=None,
decoder=None,
autoencoder=None,
latent_dim=None):
super(VAE, self).__init__(encoder=None, decoder=None)
# Encoder and decoder must be provided.
assert (encoder != None and decoder != None)
# From loading.
if encoder != None and decoder != None and autoencoder != None:
self.encoder = encoder
self.decoder = decoder
self.autoencoder = autoencoder
self.latent_dim = decoder.inputs[0].shape.as_list()[-1]
return
# Set the latent dimensions.
self.latent_dim = latent_dim
assert self.latent_dim != None
# Encoder.
encoder_input = encoder.inputs[0]
encoder_output = encoder.outputs[0]
z_mean = layers.Dense(self.latent_dim, name='z_mean')(encoder_output)
z_log_var = layers.Dense(self.latent_dim,
name='z_log_var')(encoder_output)
z = layers.Lambda(sampling, output_shape=(self.latent_dim, ),
name='z')([z_mean, z_log_var])
self.encoder = Model(encoder_input, [z_mean, z_log_var, z],
name='encoder')
# Decoder.
self.decoder = decoder
# Creating the VAE.
inputs = self.encoder.inputs[0]
outputs = self.decoder(self.encoder(inputs)[2]) # This is z.
self.autoencoder = Model(inputs, outputs, name="vae")
def get_unet_resnet(input_shape):
resnet_base = ResNet50(include_top=False,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000)
resnet_base = ResNet50(include_top=False,
weights='imagenet',
input_shape=(224, 224, 3))
if args.show_summary:
resnet_base.summary()
del resnet_base
for l in resnet_base.layers:
l.trainable = True
conv1 = resnet_base.get_layer("activation_1").output
conv2 = resnet_base.get_layer("activation_10").output
conv3 = resnet_base.get_layer("activation_22").output
conv4 = resnet_base.get_layer("activation_40").output
conv5 = resnet_base.get_layer("activation_49").output
up6 = concatenate([UpSampling2D()(conv5), conv4], axis=-1)
conv6 = conv_block_simple(up6, 256, "conv6_1")
conv6 = conv_block_simple(conv6, 256, "conv6_2")
up7 = concatenate([UpSampling2D()(conv6), conv3], axis=-1)
conv7 = conv_block_simple(up7, 192, "conv7_1")
conv7 = conv_block_simple(conv7, 192, "conv7_2")
up8 = concatenate([UpSampling2D()(conv7), conv2], axis=-1)
conv8 = conv_block_simple(up8, 128, "conv8_1")
conv8 = conv_block_simple(conv8, 128, "conv8_2")
up9 = concatenate([UpSampling2D()(conv8), conv1], axis=-1)
conv9 = conv_block_simple(up9, 64, "conv9_1")
conv9 = conv_block_simple(conv9, 64, "conv9_2")
vgg = VGG16(input_shape=(224, 224, 3),
input_tensor=resnet_base.input,
include_top=False)
for l in vgg.layers:
l.trainable = False
vgg_first_conv = vgg.get_layer("block1_conv2").output
up10 = concatenate(
[UpSampling2D()(conv9), resnet_base.input, vgg_first_conv], axis=-1)
conv10 = conv_block_simple(up10, 32, "conv10_1")
conv10 = conv_block_simple(conv10, 32, "conv10_2")
conv10 = SpatialDropout2D(0.2)(conv10)
x = Conv2D(1, (1, 1), activation="sigmoid", name="prediction")(conv10)
model = Model(resnet_base.input, x)
return model
def build(self):
mc = self.config.model
in_x = x = Input((101, 8, 8))
# (batch, channels, height, width)
x = Conv2D(filters=mc.cnn_filter_num,
kernel_size=mc.cnn_filter_size,
padding="same",
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg))(x)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
for _ in range(mc.res_layer_num):
x = self._build_residual_block(x)
res_out = x
# for policy output
x = Conv2D(filters=2,
kernel_size=1,
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg))(res_out)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
# no output for 'pass'
policy_out = Dense(self.config.n_labels,
kernel_regularizer=l2(mc.l2_reg),
activation="softmax",
name="policy_out")(x)
# for value output
x = Conv2D(filters=1,
kernel_size=1,
data_format="channels_first",
use_bias=False,
kernel_regularizer=l2(mc.l2_reg))(res_out)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
x = Dense(mc.value_fc_size,
kernel_regularizer=l2(mc.l2_reg),
activation="relu")(x)
value_out = Dense(1,
kernel_regularizer=l2(mc.l2_reg),
activation="tanh",
name="value_out")(x)
self.model = Model(in_x, [policy_out, value_out], name="chess_model")
def EEGNet(nb_classes=3,
Chans=3,
Samples=75,
dropoutRate=0.25,
kernLength=150,
F1=2,
D=2,
F2=8,
dropoutType='Dropout'):
dropoutType = Dropout
input1 = Input(shape=(1, Samples, Chans))
block1 = Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(1, Samples, Chans),
use_bias=False)(input1)
block1 = BatchNormalization(axis=1)(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.),
data_format='channels_first')(block1)
block1 = BatchNormalization(axis=1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 2), data_format='channels_first')(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16), use_bias=False,
padding='same')(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 1))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
lstm = LSTM(128, activation='tanh', recurrent_activation='hard_sigmoid', \
use_bias=True, kernel_initializer='glorot_uniform', \
recurrent_initializer='orthogonal', \
unit_forget_bias=True, kernel_regularizer=None, \
recurrent_regularizer=None, \
bias_regularizer=None, activity_regularizer=None, \
kernel_constraint=None, recurrent_constraint=None, \
bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, \
implementation=1, return_sequences=True, return_state=False, \
go_backwards=False, stateful=False, unroll=False)(block2)
dense = Dense(nb_classes, name='dense',
kernel_constraint=max_norm(0.25))(lstm)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
def __init__(self, work_dir, config, model_urls):
self.merge_extractors = []
self.input_size = 0
for model_file in model_urls:
_model = load_model(model_file)
new_model = Model(inputs=_model.input,
outputs=_model.layers[-2].output)
# assumes output is in form of (None,Dimension)
self.input_size += new_model.output.shape[1].value
self.merge_extractors.append(new_model)
EmotTrainCnnBase.__init__(self, work_dir, config=config)
def test_sparse_placeholder_fit():
test_inputs = [sparse.random(6, 3, density=0.25).tocsr() for _ in range(2)]
test_outputs = [sparse.random(6, i, density=0.25).tocsr() for i in range(3, 5)]
in1 = Input(shape=(3,))
in2 = Input(shape=(3,), sparse=True)
out1 = Dropout(0.5, name='dropout')(in1)
out2 = Dense(4, name='dense_1')(in2)
model = Model([in1, in2], [out1, out2])
model.predict(test_inputs, batch_size=2)
model.compile('rmsprop', 'mse')
model.fit(test_inputs, test_outputs,
epochs=1, batch_size=2, validation_split=0.5)
model.evaluate(test_inputs, test_outputs, batch_size=2)
def get_interlayer_output(model,
input_x,
intermediate_layer_name='intermediate_dense'):
'''
@param model: the model best has been trained
'''
intermediate_layer_model = Model(
inputs=model.input,
outputs=model.get_layer(intermediate_layer_name).output)
intermediate_x = intermediate_layer_model.predict(input_x)
return intermediate_x
def test_model_with_partial_loss():
a = Input(shape=(3, ), name='input_a')
a_2 = Dense(4, name='dense_1')(a)
dp = Dropout(0.5, name='dropout')
a_3 = dp(a_2)
model = Model(a, [a_2, a_3])
optimizer = 'rmsprop'
loss = {'dropout': 'mse'}
model.compile(optimizer, loss, metrics=['mae'])
input_a_np = np.random.random((10, 3))
output_a_np = np.random.random((10, 4))
# test train_on_batch
out = model.train_on_batch(input_a_np, output_a_np)
out = model.test_on_batch(input_a_np, output_a_np)
# fit
out = model.fit(input_a_np, [output_a_np])
# evaluate
out = model.evaluate(input_a_np, [output_a_np])
# Same without dropout.
a = Input(shape=(3, ), name='input_a')
a_2 = Dense(4, name='dense_1')(a)
a_3 = Dense(4, name='dense_2')(a_2)
model = Model(a, [a_2, a_3])
optimizer = 'rmsprop'
loss = {'dense_2': 'mse'}
model.compile(optimizer, loss, metrics={'dense_1': 'mae'})
# test train_on_batch
out = model.train_on_batch(input_a_np, output_a_np)
out = model.test_on_batch(input_a_np, output_a_np)
# fit
out = model.fit(input_a_np, [output_a_np])
# evaluate
out = model.evaluate(input_a_np, [output_a_np])
def test_trainable_weights_count_consistency():
"""Tests the trainable weights consistency check of Model.
This verifies that a warning is shown if model.trainable is modified
and the model is summarized/run without a new call to .compile()
Reproduce issue #8121
"""
a = Input(shape=(3, ), name='input_a')
model1 = Model(inputs=a, outputs=Dense(1)(a))
model1.trainable = False
b = Input(shape=(3, ), name='input_b')
y = model1(b)
model2 = Model(inputs=b, outputs=Dense(1)(y))
model2.compile(optimizer='adam', loss='mse')
model1.trainable = True
# Should warn on .summary()
with pytest.warns(UserWarning) as w:
model2.summary()
warning_raised = any(['Discrepancy' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when trainable is modified without .compile.'
# And on .fit()
with pytest.warns(UserWarning) as w:
model2.fit(x=np.zeros((5, 3)), y=np.zeros((5, 1)))
warning_raised = any(['Discrepancy' in str(w_.message) for w_ in w])
assert warning_raised, 'No warning raised when trainable is modified without .compile.'
# And shouldn't warn if we recompile
model2.compile(optimizer='adam', loss='mse')
with pytest.warns(None) as w:
model2.summary()
assert len(
w
) == 0, "Warning raised even when .compile() is called after modifying .trainable"
def build_net(self, is_train):
super(NetU_Vgg, self).build_net(is_train)
locals()['in0'] = Input(shape=(self.row_in, self.col_in, self.dep_in))
locals()['pre0'] = self.preproc(locals()['in0'], 'pre0', 0, self.fs[0],
self.act)
creater, convs = self.config[self.variation]
base_model = creater(input_tensor=locals()['pre0'],
include_top=False,
weights='imagenet' if self.pre_trained else None)
# print(base_model.summary())
for layer in base_model.layers:
layer.trainable = True # allow training on pre-trained weights
for i in range(5):
locals()['dmerge%d' %
i] = locals()['dconv%d' % i] = base_model.get_layer(
"block%d_conv%d" % (i + 1, convs[i])).output
locals()['dproc%d' %
(i + 1)] = locals()['djoin%d' % (i + 1)] = locals()[
'dsamp%d' % (i + 1)] = base_model.get_layer(
"block%d_pool" % (i + 1)).output
for i in range(len(self.fs) - 2, -1, -1):
prev_layer = locals()['dproc%d' % (i + 1)] if i == len(
self.fs) - 2 else locals()['uproc%d' % (i + 1)]
locals()['uconv%d' % (i + 1)] = self.upconv(
prev_layer, 'uconv%d' % (i + 1), i, self.fs[i + 1], self.act)
locals()['ujoin%d' % (i + 1)] = self.upjoin(
locals()['uconv%d' % (i + 1)],
locals()['djoin%d' % (i + 1)], 'ujoin%d' % (i + 1), i,
self.fs[i + 1], self.act)
locals()['usamp%d' % i] = self.upsamp(
locals()['ujoin%d' % (i + 1)], self.ps[i], 'usamp%d' % i, i,
self.fs[i + 1], self.act)
locals()['umerge%d' % i] = self.upmerge(locals()['usamp%d' % i],
locals()['dmerge%d' % i],
'umerge%d' % i, i,
self.fs[i], self.act)
locals()['uproc%d' % i] = self.upproc(locals()['umerge%d' % i],
'uproc%d' % i, i, self.fs[i],
self.act)
locals()['post0'] = self.postproc(locals()['uproc0'], 'post0', 0,
self.fs[0], self.act)
locals()['out0'] = cvac(locals()['post0'],
'out0',
0,
self.dep_out,
self.out,
size=1)
self.net = Model(locals()['in0'], locals()['out0'])
def build_network(self):
""" Build the Policy Value Neural Net using Keras. """
inputs = Input(shape=(4, self.size, self.size))
# 3 common conv layers
c_conv1 = Conv2D(filters=32,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(inputs)
c_conv2 = Conv2D(filters=64,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(c_conv1)
c_conv3 = Conv2D(filters=128,
kernel_size=(3, 3),
padding="same",
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(c_conv2)
# policy head
p_conv = Conv2D(filters=4,
kernel_size=(1, 1),
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(c_conv3)
p_flat = Flatten()(p_conv)
self.policy_net = Dense(self.size * self.size,
activation="softmax",
kernel_regularizer=l2(self.l2_const))(p_flat)
# value head
v_conv = Conv2D(filters=2,
kernel_size=(1, 1),
data_format="channels_first",
activation="relu",
kernel_regularizer=l2(self.l2_const))(c_conv3)
v_flat = Flatten()(v_conv)
v_dense = Dense(64, kernel_regularizer=l2(self.l2_const))(v_flat)
self.value_net = Dense(1,
activation="tanh",
kernel_regularizer=l2(self.l2_const))(v_dense)
# connect and build the model
self.model = Model(inputs, [self.policy_net, self.value_net])
losses = ['categorical_crossentropy', 'mean_squared_error']
self.model.compile(optimizer=Adam(), loss=losses)
def g():
seq = Input(shape=(input_size, nb_chars))
z = Input(shape=(z_size, ))
z_rep = RepeatVector(input_size)(z)
seq_and_z = merge([seq, z_rep], mode='concat', concat_axis=-1)
fake_prob = sequential([
LSTM(8),
RepeatVector(output_size),
LSTM(8, return_sequences=True),
TimeDistributed(Dense(nb_chars, activation='softmax')),
])(seq_and_z)
g = Model([z, seq], [fake_prob])
return g
def build_model():
"""
builds full keras model and returns it
"""
in_x = x = Input((1, 8, 8))
# (batch, channels, height, width)
x = Conv2D(filters=cnn_filter_num, kernel_size=cnn_first_filter_size, padding="same", data_format="channels_first",
use_bias=False, kernel_regularizer=l2(l2_reg),
name="input_conv-" + str(cnn_first_filter_size) + "-" + str(cnn_filter_num))(x)
x = BatchNormalization(axis=1, name="input_batchnorm")(x)
x = Activation("relu", name="input_relu")(x)
for i in range(res_layer_num):
x = _build_residual_block(x, i + 1)
res_out = x
# for policy output
x = Conv2D(filters=2, kernel_size=1, data_format="channels_first", use_bias=False, kernel_regularizer=l2(l2_reg),
name="policy_conv-1-2")(res_out)
x = BatchNormalization(axis=1, name="policy_batchnorm")(x)
x = Activation("relu", name="policy_relu")(x)
x = Flatten(name="policy_flatten")(x)
# no output for 'pass'
policy_out = Dense(n_labels, kernel_regularizer=l2(l2_reg), activation="softmax", name="policy_out")(x)
# for value output
x = Conv2D(filters=4, kernel_size=1, data_format="channels_first", use_bias=False, kernel_regularizer=l2(l2_reg),
name="value_conv-1-4")(res_out)
x = BatchNormalization(axis=1, name="value_batchnorm")(x)
x = Activation("relu", name="value_relu")(x)
x = Flatten(name="value_flatten")(x)
x = Dense(value_fc_size, kernel_regularizer=l2(l2_reg), activation="relu", name="value_dense")(x)
value_out = Dense(1, kernel_regularizer=l2(l2_reg), activation="tanh", name="value_out")(x)
model = Model(in_x, [policy_out, value_out], name="hex_model")
sgd = optimizers.SGD(lr=learning_rate, momentum=momentum)
losses = ['categorical_crossentropy', 'mean_squared_error']
model.compile(loss=losses, optimizer='adam', metrics=['accuracy', 'mae'])
model.summary()
return model
def decoder_resnet(label_sizes,
nb_filter=16,
data_shape=(1, 64, 64),
nb_bits=12,
resnet_depth=(3, 4, 6, 3),
optimizer='adam'):
def _bn_relu_conv(nb_filter, nb_row=3, nb_col=3, subsample=1):
return sequential([
BatchNormalization(mode=0, axis=1),
ELU(),
Convolution2D(nb_filter=nb_filter,
nb_row=nb_row,
nb_col=nb_col,
subsample=(subsample, subsample),
init="he_normal",
border_mode="same")
])
def f(nb_filter, subsample=1):
return sequential([
_bn_relu_conv(nb_filter, subsample=subsample),
_bn_relu_conv(nb_filter),
])
input = Input(shape=data_shape)
fitlers_by_depth = [nb_filter * 2**i for i in range(len(resnet_depth))]
print("fitlers_by_depth", fitlers_by_depth)
x = _bn_relu_conv(nb_filter, 3, 3, subsample=2)(input)
for i, (n, d) in enumerate(zip(fitlers_by_depth, resnet_depth)):
for di in range(d):
if di == 0 and i != 0:
shortcut = _bn_relu_conv(n, 1, 1, subsample=2)
subsample = 2
else:
shortcut = lambda x: x
subsample = 1
x = merge([shortcut(x), f(n, subsample)(x)], mode='sum')
outputs, losses = decoder_end_block(x,
label_sizes,
nb_bits,
activation=lambda: ELU())
model = Model(input, list(outputs.values()))
model.compile(
optimizer,
loss=list(losses.values()),
loss_weights={k: decoder_loss_weights(k)
for k in losses.keys()})
return model
def simple_model(self, layers):
print("building simple input")
input = x = Input(shape=(12, 8, 8))
x = BatchNormalization(name="in_2", axis=1)(x)
x = Conv2D(padding='same',
filters=64,
kernel_size=4,
use_biase=False,
data_format="channels_first",
name="in_1")
x = Activation("relu", name="in_3")(x)
for i in range(layers):
self.build_residuals(x, i)
return Model(input, [out1, out2], name="palmtree")
def _build_network(self):
# Input_Layer
init_x = Input((3, self._board_size, self._board_size))
x = init_x
# Convolutional Layer
x = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 1),
padding='same',
data_format='channels_first',
kernel_regularizer=l2(self._l2_coef))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
# Residual Layer
x = self._residual_block(x)
x = self._residual_block(x)
x = self._residual_block(x)
# Policy Head
policy = Conv2D(filters=2,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
data_format='channels_first',
kernel_regularizer=l2(self._l2_coef))(x)
policy = BatchNormalization()(policy)
policy = Activation('relu')(policy)
policy = Flatten()(policy)
policy = Dense(self._board_size * self._board_size,
kernel_regularizer=l2(self._l2_coef))(policy)
self._policy = Activation('softmax')(policy)
# Value Head
value = Conv2D(filters=1,
kernel_size=(1, 1),
strides=(1, 1),
padding='same',
data_format="channels_first",
kernel_regularizer=l2(self._l2_coef))(x)
value = BatchNormalization()(value)
value = Activation('relu')(value)
value = Flatten()(value)
value = Dense(32, kernel_regularizer=l2(self._l2_coef))(value)
value = Activation('relu')(value)
value = Dense(1, kernel_regularizer=l2(self._l2_coef))(value)
self._value = Activation('tanh')(value)
# Define Network
self._model = Model(inputs=init_x, outputs=[self._policy, self._value])
# Define the Loss Function
opt = SGD(lr=self._lr, momentum=self._momentum, nesterov=True)
losses_type = ['categorical_crossentropy', 'mean_squared_error']
self._model.compile(optimizer=opt, loss=losses_type)
def build_model():
#X_train, X_test, Y_train, noLSTM, train_labels
#X_train, X_test, y_train, np.array(archs[q]), train_labels
Chans = X_train.shape[1]
Samples = slidingWindowSize
input_main = Input((1, Chans, slidingWindowSize))
block1 = Conv2D(25, (1, 5),
input_shape=(1, Chans, Samples),
kernel_constraint=max_norm(2.))(input_main)
block1 = Conv2D(25, (Chans, 1), kernel_constraint=max_norm(2.))(block1)
block1 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block1)
block1 = Activation('elu')(block1)
block1 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block1)
block1 = Dropout(0.2)(block1)
block2 = Conv2D(50, (1, 5), kernel_constraint=max_norm(2.))(block1)
block2 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block2)
block2 = Activation('elu')(block2)
block2 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block2)
block2 = Dropout(0.2)(block2)
block3 = Conv2D(100, (1, 5), kernel_constraint=max_norm(2.))(block2)
block3 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block3)
block3 = Activation('elu')(block3)
block3 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block3)
block3 = Dropout(0.2)(block3)
block4 = Conv2D(200, (1, 5), kernel_constraint=max_norm(2.))(block3)
block4 = BatchNormalization(axis=1, epsilon=1e-05, momentum=0.1)(block4)
block4 = Activation('elu')(block4)
block4 = MaxPooling2D(pool_size=(1, 2), strides=(1, 2))(block4)
block4 = Dropout(0.2)(block4)
flatten = Flatten()(block4)
dense = Dense(3, kernel_constraint=max_norm(0.5))(flatten)
softmax = Activation('softmax')(dense)
# ['acc', 'loss', 'val_acc', 'val_loss']
model = Model(inputs=input_main, outputs=softmax)
opt = optimizers.Adam(lr=0.0003,
beta_1=0.91,
beta_2=0.999,
epsilon=1e-08,
decay=0.0,
amsgrad=False)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=['accuracy'])
def build_model(args):
cnn_filter_num = args['cnn_filter_num']
cnn_filter_size = args['cnn_filter_size']
l2_reg = args['l2_reg']
in_x = x = Input(args['input_dim'])
# (batch, channels, height, width)
x = Conv2D(filters=cnn_filter_num,
kernel_size=cnn_filter_size,
padding="same",
data_format="channels_first",
kernel_regularizer=l2(l2_reg))(x)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
for _ in range(args['res_layer_num']):
x = _build_residual_block(args, x)
res_out = x
# for policy output
x = Conv2D(filters=2,
kernel_size=1,
data_format="channels_first",
kernel_regularizer=l2(l2_reg))(res_out)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
policy_out = Dense(args['policy_dim'],
kernel_regularizer=l2(l2_reg),
activation="softmax",
name="policy")(x)
# for value output
x = Conv2D(filters=1,
kernel_size=1,
data_format="channels_first",
kernel_regularizer=l2(l2_reg))(res_out)
x = BatchNormalization(axis=1)(x)
x = Activation("relu")(x)
x = Flatten()(x)
x = Dense(256, kernel_regularizer=l2(l2_reg), activation="relu")(x)
value_out = Dense(1,
kernel_regularizer=l2(l2_reg),
activation="tanh",
name="value")(x)
return Model(in_x, [policy_out, value_out], name="model")
def EEGNet(nb_classes=3,
Chans=5,
Samples=300,
dropoutRate=0.25,
kernLength=150,
F1=2,
D=2,
F2=8,
dropoutType='Dropout'):
if dropoutType == 'SpatialDropout2D':
dropoutType = SpatialDropout2D
elif dropoutType == 'Dropout':
dropoutType = Dropout
else:
raise ValueError('dropoutType must be one of SpatialDropout2D '
'or Dropout, passed as a string.')
input1 = Input(shape=(1, Samples, Chans))
block1 = Conv2D(F1, (1, kernLength),
padding='same',
input_shape=(1, Samples, Chans),
use_bias=False)(input1)
block1 = BatchNormalization(axis=1)(block1)
block1 = DepthwiseConv2D((Chans, 1),
use_bias=False,
depth_multiplier=D,
depthwise_constraint=max_norm(1.),
data_format='channels_first')(block1)
block1 = BatchNormalization(axis=1)(block1)
block1 = Activation('elu')(block1)
block1 = AveragePooling2D((1, 2), data_format='channels_first')(block1)
block1 = dropoutType(dropoutRate)(block1)
block2 = SeparableConv2D(F2, (1, 16), use_bias=False,
padding='same')(block1)
block2 = BatchNormalization(axis=1)(block2)
block2 = Activation('elu')(block2)
block2 = AveragePooling2D((1, 1))(block2)
block2 = dropoutType(dropoutRate)(block2)
flatten = Flatten(name='flatten')(block2)
dense = Dense(nb_classes, name='dense',
kernel_constraint=max_norm(0.25))(flatten)
softmax = Activation('softmax', name='softmax')(dense)
return Model(inputs=input1, outputs=softmax)
