def create_model(self, input_shape, output_num):
# This returns a tensor
inputs = Input(shape=input_shape)
# a layer instance is callable on a tensor, and returns a tensor
x = Conv2D(12, (4, 4), input_shape=input_shape, data_format="channels_last", padding='same', activation='relu', kernel_initializer='he_normal')(inputs)
x = Conv2D(24, (4, 4), padding='same', activation='relu', kernel_initializer='he_normal')(x)
x = Conv2D(48, (4, 4), padding='same', activation='relu', kernel_initializer='he_normal')(x)
x = Flatten()(x)
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(48, activation='relu', kernel_initializer='he_normal')(x)
value = Dense(32, activation='relu', kernel_initializer='he_normal')(x)
value = Dense(1, activation=self.fin_activation, kernel_initializer='he_normal')(value)
advantage = Dense(32, activation='relu', kernel_initializer='he_normal')(x)
advantage = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(advantage)
mean = Lambda(lambda x: K.mean(x, axis=1, keepdims=True))(advantage)
advantage = Subtract()([advantage, mean])
predictions = Add()([value, advantage])
# This creates a model that includes
# the Input layer and the stacked output layers
model = FuncModel(inputs=inputs, outputs=predictions)
# model.compile happens in baseclass method compile_model()
return model
def create_model(self, input_shape, output_num):
# This returns a tensor
inputs = Input(shape=input_shape)
# a layer instance is callable on a tensor, and returns a tensor
x = Conv2D(12, (4, 4), input_shape=input_shape, data_format="channels_last", padding='same', activation='relu', kernel_initializer='he_normal')(inputs)
x = Conv2D(24, (4, 4), padding='same', activation='relu', kernel_initializer='he_normal')(x)
x = Conv2D(48, (4, 4), padding='same', activation='relu', kernel_initializer='he_normal')(x)
x = Flatten()(x)
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(48, activation='relu', kernel_initializer='he_normal')(x)
# network separate state value and advantages
value_fc = Dense(32, activation='relu', kernel_initializer='he_normal')(x)
value = Dense(1, activation=self.fin_activation)(value_fc)
value = Lambda(lambda s: K.expand_dims(s[:, 0], -1),
output_shape=(output_num,))(value)
advantage_fc = Dense(32, activation='relu', kernel_initializer='he_normal')(x)
advantage = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(advantage_fc)
advantage = Lambda(lambda a: a[:, :] - K.mean(a[:, :], keepdims=True),
output_shape=(output_num,))(advantage)
q_value = add([value, advantage])
# This creates a model that includes
# the Input layer and the stacked output layers
model = FuncModel(inputs=inputs, outputs=q_value)
# model.compile happens in baseclass method compile_model()
return model
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
inputs = Input(shape=input_shape)
advantages = Input(shape=[1])
x = self.conv_layer(inputs, 60 * multipl, (3, 3))
x = self.conv_layer(inputs, 60 * multipl, (3, 3))
x = Flatten()(x)
x = Dense(128 * multipl, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(x)
probs = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(x)
def custom_loss(y_true, y_pred):
out = K.clip(y_pred, 1e-8, 1 - 1e-8)
log_liklyhood = y_true * K.log(out)
return K.sum(-log_liklyhood * advantages)
policy = FuncModel(inputs=[inputs, advantages], outputs=[probs]) # for training
policy.compile(optimizer=self.opt, loss=custom_loss)
predict = FuncModel(inputs=[inputs], outputs=[probs]) # for using
predict.compile(optimizer=self.opt, loss='mse')
return {'policy_model': policy, 'predict_model': predict, 'compiled': True}
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
inputs = Input(shape=input_shape)
#advantages = Input(shape=[1])
x = self.conv_layer(inputs, 50 * multipl, (3, 3))
x = self.residual_layer(x, 75 * multipl, (3, 3))
#x = self.residual_layer(x, 75 * multipl, (3, 3))
#x = self.residual_layer(x, 75 * multipl, (3, 3))
x = Flatten()(x)
act = Dense(128 * multipl, use_bias=False, kernel_initializer='he_normal')(x)
act = BatchNormalization()(act)
act = LeakyReLU()(act)
#act = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(act)
act_predictions = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(act)
crit = Dense(128 * multipl, use_bias=False, kernel_initializer='he_normal')(x)
crit = BatchNormalization()(crit)
crit = LeakyReLU()(crit)
#crit = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(crit)
crit_predictions = Dense(1, activation='linear', kernel_initializer='he_normal')(crit)
def custom_loss(y_true, y_pred):
out = K.clip(y_pred, 1e-8, 1 - 1e-8)
log_liklyhood = y_true * K.log(out)
return K.sum(-log_liklyhood * advantages)
#actor = FuncModel(inputs=[inputs, advantages], outputs=act_predictions) # for training
actor = FuncModel(inputs=[inputs], outputs=act_predictions) # for training
critic = FuncModel(inputs=inputs, outputs=crit_predictions) # for training
predict = FuncModel(inputs=inputs, outputs=act_predictions) # for using
# actor.compile(optimizer=self.opt[0], loss=custom_loss)
# critic.compile(optimizer=self.opt[1], loss='mse')
# predict.compile(optimizer=self.opt[1], loss='mse')
return {'actor_model': actor, 'critic_model': critic, 'predict_model': predict, 'compiled': True}
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
# This returns a tensor
inputs = Input(shape=input_shape)
x = Flatten()(inputs) # converts the 3D feature maps to 1D feature vectors
x = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(x)
predictions = Dense(output_num, activation=self.fin_activation)(x)
model = FuncModel(inputs=inputs, outputs=predictions)
# model.compile happens in baseclass method compile_model()
return model
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
inputs = Input(shape=input_shape)
x = Conv2D(24 * multipl, (3, 3), input_shape=input_shape, use_bias=False, data_format="channels_last", padding='same', kernel_initializer='he_normal')(inputs)
x = BatchNormalization()(x)
#block_1_output = Activation('relu')(x)
block_1_output = LeakyReLU(alpha=0.2)(x)
x = Conv2D(24 * multipl, (3, 3), use_bias=False, padding='same', kernel_initializer='he_normal')(block_1_output)
x = BatchNormalization()(x)
#x = Activation('relu')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Add()([x, block_1_output])
x = Conv2D(48 * multipl, (3, 3), padding='same', use_bias=False, kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
#block_2_output = Activation('relu')(x)
block_2_output = LeakyReLU(alpha=0.2)(x)
x = Conv2D(48 * multipl, (3, 3), padding='same', use_bias=False, kernel_initializer='he_normal')(block_2_output)
x = BatchNormalization()(x)
#x = Activation('relu')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Add()([x, block_2_output])
x = Conv2D(96 * multipl, (3, 3), padding='same', use_bias=False, kernel_initializer='he_normal')(x)
x = BatchNormalization()(x)
#block_3_output = Activation('relu')(x)
block_3_output = LeakyReLU(alpha=0.2)(x)
x = Conv2D(96 * multipl, (3, 3), padding='same', use_bias=False, kernel_initializer='he_normal')(block_3_output)
x = BatchNormalization()(x)
#x = Activation('relu')(x)
x = LeakyReLU(alpha=0.2)(x)
x = Add()([x, block_3_output])
x = Flatten()(x)
act = Dense(128 * multipl, activation='relu', kernel_initializer='he_normal')(x)
act = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(act)
act_predictions = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(act)
crit = Dense(128 * multipl, activation='relu', kernel_initializer='he_normal')(x)
crit = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(crit)
crit_predictions = Dense(1, activation='linear', kernel_initializer='he_normal')(crit)
model = FuncModel(inputs=inputs, outputs=[act_predictions, crit_predictions])
return {'model': model}
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
# This returns a tensor
inputs = Input(shape=input_shape)
# a layer instance is callable on a tensor, and returns a tensor
# This returns a tensor
inputs = Input(shape=input_shape)
x = Flatten()(inputs)
x = Dense(24 * multipl, activation='relu', kernel_initializer='he_normal')(x)
predictions = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(x)
# This creates a model that includes
# the Input layer and the stacked output layers
model = FuncModel(inputs=inputs, outputs=predictions)
return model
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
inputs = Input(shape=input_shape)
x = self.conv_layer(inputs, 75 * multipl, (3, 3))
x = self.residual_layer(x, 75 * multipl, (3, 3))
#x = self.residual_layer(x, 75 * multipl, (3, 3))
#x = self.residual_layer(x, 75 * multipl, (3, 3))
x = Flatten()(x)
act = Dense(128 * multipl, activation='relu', kernel_initializer='he_normal')(x)
act = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(act)
act_predictions = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(act)
crit = Dense(128 * multipl, activation='relu', kernel_initializer='he_normal')(x)
crit = Dense(64 * multipl, activation='relu', kernel_initializer='he_normal')(crit)
crit_value = Dense(1, activation='tanh', kernel_initializer='he_normal')(crit)
model = FuncModel(inputs=inputs, outputs=[act_predictions, crit_value])
return {'model': model, 'compiled': False, 'loss_fns': [self.loss_fn1, self.loss_fn2]}
def create_model(self, input_shape, output_num):
# This returns a tensor
inputs = Input(shape=input_shape)
x = Flatten()(inputs) # converts the 3D feature maps to 1D feature vectors
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
x = Dense(64, activation='relu', kernel_initializer='he_normal')(x)
value = Dense(1, activation=self.fin_activation)(x)
advantage = Dense(output_num, activation=self.fin_activation)(x)
mean = Lambda(lambda x: K.mean(x, axis=1, keepdims=True))(advantage)
advantage = Subtract()([advantage, mean])
predictions = Add()([value, advantage])
# This creates a model that includes
# the Input layer and the stacked output layers
model = FuncModel(inputs=inputs, outputs=predictions)
# model.compile happens in baseclass method compile_model()
return model
def update_model(self, model, analyze_layers=None):
"""Updates the model with new weights (for instance during training) """
if analyze_layers is None:
analyze_layers = self.analyze_layers
layer_outputs = [
model.layers[layer_num].output for layer_num in analyze_layers
if layer_num <= len(model.layers) - 1
] # Extracts the outputs of the top x layers
# remove flatten layer (if present)
for idx, layers in enumerate(layer_outputs):
if layers.name.startswith('flatten'):
del layer_outputs[idx]
break
# add final (two) layer(s) (if not present)
final_layer = model.layers[len(model.layers) - 2]
if final_layer.output.name != layer_outputs[-2].name:
layer_outputs.append(final_layer.output)
final_layer = model.layers[len(model.layers) - 1]
if final_layer.output.name != layer_outputs[-1].name:
layer_outputs.append(final_layer.output)
self.activation_model = FuncModel(
inputs=model.input, outputs=layer_outputs
) # Creates a model that will return these outputs, given the model input
self.activation_model.name2 = "analyze_activation_model"
# get layernames
self.activationlayer_names = []
for layer in layer_outputs:
self.activationlayer_names.append(layer.name)
# other
self.model_name = model.model_name
self.n_features = [] # number of features per layer
def create_model(self, input_shape, output_num):
multipl = self.layer_multiplier
# This returns a tensor
inputs = Input(shape=input_shape)
# a layer instance is callable on a tensor, and returns a tensor
x = Conv2D(12 * multipl, (3, 3), input_shape=input_shape, data_format="channels_last", padding='same', activation='tanh', kernel_initializer='he_normal')(inputs)
x = Conv2D(24 * multipl, (3, 3), padding='valid', activation='tanh', kernel_initializer='he_normal')(x)
x = Conv2D(48 * multipl, (3, 3), padding='valid', activation='tanh', kernel_initializer='he_normal')(x)
x = Flatten()(x)
x = Dense(48 * multipl, activation='relu', kernel_initializer='he_normal')(x)
# x = Dense(32 * multipl, activation='relu', kernel_initializer='he_normal')(x)
predictions = Dense(output_num, activation=self.fin_activation, kernel_initializer='he_normal')(x)
# This creates a model that includes
# the Input layer and the stacked output layers
model = FuncModel(inputs=inputs, outputs=predictions)
# model.compile(optimizer='rmsprop',
# loss='categorical_crossentropy',
# metrics=['accuracy'])
# model.compile happens in baseclass method compile_model()
return {'model': model}