def create_model(self):
model = Sequential()
model.add(
Dense(self.neurons,
input_dim=self.initial,
activation='relu',
kernel_initializer=lecun_uniform(seed=self.seed)))
for i in range(self.hidden_layers - 1):
model.add(
Dense(self.neurons,
activation='relu',
kernel_initializer=lecun_uniform(seed=self.seed)))
model.add(
Dense(self.final,
activation='softmax',
kernel_initializer=lecun_uniform(seed=self.seed)))
opt = optimizers.SGD(lr=self.learning_rate, decay=self.lr_decay)
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model
def build(self):
"""
Builds the neural network using keras, and stores the model in self.model.
Uses shape parameters from init and the learning rate self.lr.
You may change this, though what is given should be a good start.
"""
model = Sequential()
#model.add(Dense(self.num_hidden1, init='lecun_uniform', input_shape=(self.num_inputs,)))
model.add(
Dense(self.num_hidden1,
init=lecun_uniform(seed=xxx),
input_shape=(self.num_inputs, )))
model.add(Activation('relu'))
#model.add(Dense(self.num_hidden2, init='lecun_uniform'))
model.add(
Dense(self.num_hidden2,
init=lecun_uniform(seed=xxx),
input_shape=(self.num_inputs, )))
model.add(Activation('relu'))
#model.add(Dense(self.num_output, init='lecun_uniform'))
model.add(Dense(self.num_output, init=lecun_uniform(seed=xxx)))
model.add(Activation('linear'))
rms = RMSprop(lr=self.lr)
model.compile(loss='mse', optimizer=rms)
self.model = model
def BidirectionalRNN(input_shape):
model = Sequential()
model.add(
Bidirectional(LSTM(32,
kernel_initializer=initializers.lecun_uniform(35),
return_sequences=True),
input_shape=input_shape))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Bidirectional(
LSTM(64,
kernel_initializer=initializers.lecun_uniform(55),
return_sequences=True)))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(
Bidirectional(
LSTM(32,
kernel_initializer=initializers.lecun_uniform(35),
return_sequences=False)))
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(5, activation="softmax"))
return model
def build_model(self):
"""Build an actor (policy) network that maps states -> actions."""
# Define input layer (states)
states = layers.Input(shape=(self.state_size, ), name='states')
# Add hidden layers
net = layers.Dense(units=400,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2),
bias_regularizer=regularizers.l1())(states)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
net = layers.Dense(units=300,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2),
bias_regularizer=regularizers.l1())(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Add final output layer with sigmoid activation
raw_actions = layers.Dense(
units=self.action_size,
activation='tanh',
name='raw_actions',
kernel_initializer=initializers.RandomUniform(-0.003, 0.003),
bias_initializer=initializers.RandomUniform(-0.003, 0.003))(net)
# Scale output for each action dimension to proper range
actions = layers.Lambda(lambda x:
(x * self.action_range) + self.action_low,
name='actions')(raw_actions)
# Create Keras model
self.model = models.Model(inputs=states, outputs=actions)
# Define loss function using action value (Q value) gradients
action_gradients = layers.Input(shape=(self.action_size, ))
loss = K.mean(-action_gradients * actions)
# Incorporate any additional losses here (e.g. from regularizers)
# Define optimizer and training function
optimizer = optimizers.Adam(lr=self.lr)
updates_op = optimizer.get_updates(params=self.model.trainable_weights,
loss=loss)
self.train_fn = K.function(
inputs=[self.model.input, action_gradients,
K.learning_phase()],
outputs=[],
updates=updates_op)
def create_actor_network(self, S):
h0 = Dense(400, activation="relu",
kernel_initializer=lecun_uniform())(S)
h1 = Dense(300, activation="relu",
kernel_initializer=lecun_uniform())(h0)
V = Dense(self.a_dim[0],
activation="tanh",
kernel_initializer=RandomUniform(minval=-3e-3, maxval=3e-3),
bias_initializer=RandomUniform(minval=-3e-3,
maxval=3e-3))(h1)
return V
def create_actor_network(self, S, G=None, M=None):
input = concatenate([multiply([subtract([S, G]), M]), S])
h0 = Dense(400, activation="relu",
kernel_initializer=lecun_uniform())(input)
h1 = Dense(300, activation="relu",
kernel_initializer=lecun_uniform())(h0)
V = Dense(self.a_dim[0],
activation="tanh",
kernel_initializer=RandomUniform(minval=-3e-3, maxval=3e-3),
bias_initializer=RandomUniform(minval=-3e-3, maxval=3e-3))(h1)
return V
def cnn_model():
# create model
model = Sequential()
BatchNormalization()
model.add(
ZeroPadding2D(padding=(2, 2),
data_format=None,
input_shape=(1, 28, 28)))
model.add(Conv2D(32, (5, 5), activation='relu', use_bias=False))
BatchNormalization(axis=-1)
model.add(Conv2D(32, (5, 5), activation='relu', use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.3))
model.add(Conv2D(64, (3, 3), activation='relu', use_bias=False))
BatchNormalization(axis=-1)
model.add(Conv2D(64, (3, 3), activation='relu', use_bias=False))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.3))
model.add(Flatten())
BatchNormalization()
model.add(
Dense(512,
activation='selu',
kernel_initializer=lecun_uniform(seed=None),
use_bias=True,
bias_initializer=lecun_uniform(seed=None)))
BatchNormalization()
model.add(AlphaDropout(0.50))
model.add(
Dense(100,
activation='selu',
kernel_initializer=lecun_uniform(seed=None),
use_bias=True,
bias_initializer=lecun_uniform(seed=None)))
model.add(AlphaDropout(0.50))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
return model
def build_initializer(type, kerasDefaults, seed=None, constant=0.):
if type == 'constant':
return initializers.Constant(value=constant)
elif type == 'uniform':
return initializers.RandomUniform(
minval=kerasDefaults['minval_uniform'],
maxval=kerasDefaults['maxval_uniform'],
seed=seed)
elif type == 'normal':
return initializers.RandomNormal(mean=kerasDefaults['mean_normal'],
stddev=kerasDefaults['stddev_normal'],
seed=seed)
# Not generally available
# elif type == 'glorot_normal':
# return initializers.glorot_normal(seed=seed)
elif type == 'glorot_uniform':
return initializers.glorot_uniform(seed=seed)
elif type == 'lecun_uniform':
return initializers.lecun_uniform(seed=seed)
elif type == 'lecun_normal':
return initializers.lecun_normal(seed=seed)
elif type == 'he_normal':
return initializers.he_normal(seed=seed)
def create_base_network(input_shapes):
'''base network model - CNN
'''
#update based on real time execution and optimization
input = Input(shape=input_shapes)
conv1 = Conv2D(64,
kernel_size=3,
activation='relu',
input_shape=input_shape,
kernel_regularizer=l2(1e-4))(input)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(128,
kernel_size=3,
activation='relu',
kernel_regularizer=l2(2e-4))(pool1)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(256,
kernel_size=3,
activation='relu',
kernel_regularizer=l2(2e-4))(pool2)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
# conv4 = Conv2D(256,kernel_size=3,activation='relu', kernel_regularizer = l2(2e-4))(pool3)
# pool4 = MaxPooling2D(pool_size=(2,2))(conv4)
# conv5 = Conv2D(256,kernel_size=3,activation='relu')(pool4)
# pool5 = MaxPooling2D(pool_size=(2,2))(conv5)
flat = Flatten()(pool3)
Dense1 = Dense(2048,
activation="sigmoid",
kernel_regularizer=l2(1e-3),
bias_initializer=lecun_uniform())(flat)
return Model(input, Dense1)
def test_lecun_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
_runner(initializers.lecun_uniform(),
tensor_shape,
target_mean=0.,
target_std=std)
def create_V2A_network(A_dim, V_dim):
A_input = Input(shape=A_dim)
AP = AveragePooling1D(pool_size=pool, strides=stride,
padding='valid')(A_input)
V_input = Input(shape=V_dim)
VP = AveragePooling1D(pool_size=pool, strides=stride,
padding='valid')(V_input)
VL = LSTM(units=1024,
return_sequences=True,
stateful=False,
dropout=0.2,
recurrent_dropout=0.2,
kernel_initializer=initializers.lecun_normal(),
recurrent_initializer=initializers.lecun_uniform())(VP)
VL = TimeDistributed(
Dense(units=128,
kernel_initializer=initializers.lecun_normal(),
activation='tanh'))(VL)
VT = TimeDistributed(
Dense(units=128,
kernel_initializer=initializers.lecun_normal(),
activation='tanh'))(VP)
VL = Average()([VL, VT])
distance = Lambda(QQ, output_shape=[L, 128])([VL, AP])
res_model = Model(inputs=[A_input, V_input], outputs=distance)
#my_model.summary()
return res_model
def create_critic_network(self, S, A, G=None, M=None):
input = concatenate([multiply([subtract([S, G]), M]), S])
L1 = Dense(400, activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))
L1out = L1(input)
L1out = concatenate([L1out, A])
L2 = Dense(300, activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))
L2out = L2(L1out)
L3 = Dense(1, activation='linear',
kernel_initializer=RandomUniform(minval=-3e-4, maxval=3e-4),
kernel_regularizer=l2(0.01),
bias_initializer=RandomUniform(minval=-3e-4, maxval=3e-4))
qval = L3(L2out)
return [L1, L2, L3], qval
def create_critic_network(self, S):
h1 = Dense(400,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))(S)
h2 = Dense(300,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))(h1)
Q_values = Dense(self.num_actions * self.num_tasks,
activation='linear',
kernel_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4),
kernel_regularizer=l2(0.01),
bias_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4))(h2)
return Q_values
def build_initializer(type, kerasDefaults, seed=None, constant=0.):
""" Set the initializer to the appropriate Keras initializer function
based on the input string and learning rate. Other required values
are set to the Keras default values
Parameters
----------
type : string
String to choose the initializer
Options recognized: 'constant', 'uniform', 'normal',
'glorot_uniform', 'lecun_uniform', 'he_normal'
See the Keras documentation for a full description of the options
kerasDefaults : list
List of default parameter values to ensure consistency between frameworks
seed : integer
Random number seed
constant : float
Constant value (for the constant initializer only)
Return
----------
The appropriate Keras initializer function
"""
if type == 'constant':
return initializers.Constant(value=constant)
elif type == 'uniform':
return initializers.RandomUniform(
minval=kerasDefaults['minval_uniform'],
maxval=kerasDefaults['maxval_uniform'],
seed=seed)
elif type == 'normal':
return initializers.RandomNormal(mean=kerasDefaults['mean_normal'],
stddev=kerasDefaults['stddev_normal'],
seed=seed)
# Not generally available
# elif type == 'glorot_normal':
# return initializers.glorot_normal(seed=seed)
elif type == 'glorot_uniform':
return initializers.glorot_uniform(seed=seed)
elif type == 'lecun_uniform':
return initializers.lecun_uniform(seed=seed)
elif type == 'he_normal':
return initializers.he_normal(seed=seed)
def create_critic_network(self, S, V):
if self.network == '0':
L1 = concatenate([multiply([subtract([S, G]), M]), S])
L2 = Dense(400,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))(L1)
L3 = Dense(300,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))(L2)
Q_values = Dense(self.env.action_dim,
activation='linear',
kernel_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4),
kernel_regularizer=l2(0.01),
bias_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4))(L3)
else:
L1 = Dense(200,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))
L2 = Dense(300,
activation="relu",
kernel_initializer=lecun_uniform(),
kernel_regularizer=l2(0.01))
i1 = multiply([subtract([S, G]), M])
i2 = S
h1 = L1(i1)
h2 = L1(i2)
h3 = concatenate([h1, h2])
h4 = L2(h3)
Q_values = Dense(self.env.action_dim,
activation='linear',
kernel_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4),
kernel_regularizer=l2(0.01),
bias_initializer=RandomUniform(minval=-3e-4,
maxval=3e-4))(h4)
return Q_values
def getInitializer(init_name, learning_rate, opt, functions):
if init_name == "rnormal":
init = initializers.RandomNormal()
elif init_name == "runiform":
init = initializers.RandomUniform()
elif init_name == "varscaling":
init = initializers.VarianceScaling()
elif init_name == "orth":
init = initializers.Orthogonal()
elif init_name == "id":
init = initializers.Identity()
elif init_name == "lecun_uniform":
init = initializers.lecun_uniform()
elif init_name == "glorot_normal":
init = initializers.glorot_normal()
elif init_name == "glorot_uniform":
init = initializers.glorot_uniform()
elif init_name == "he_normal":
init = initializers.he_normal()
elif init_name == "he_uniform":
init = initializers.he_uniform()
if opt == "Adam":
optimizer = optimizers.Adam(lr=learning_rate)
elif opt == "Adagrad":
optimizer = optimizers.Adagrad(lr=learning_rate)
elif opt == "Adadelta":
optimizer = optimizers.Adadelta(lr=learning_rate)
elif opt == "Adamax":
optimizer = optimizers.Adamax(lr=learning_rate)
elif opt == "Nadam":
optimizer = optimizers.Nadam(lr=learning_rate)
elif opt == "sgd":
optimizer = optimizers.SGD(lr=learning_rate)
elif opt == "RMSprop":
optimizer = optimizers.RMSprop(lr=learning_rate)
if functions.startswith("maxout"):
functions, maxout_k = functions.split("-")
maxout_k = int(maxout_k)
else:
maxout_k = 3
if functions.startswith("leakyrelu"):
if "-" in functions:
functions, maxout_k = functions.split("-")
maxout_k = float(maxout_k)
else:
maxout_k = 0.01
return init, optimizer, functions, maxout_k
#################### VarianceScaling ####################
pytest.param(
initializers.glorot_normal(), dict(class_name="glorot_normal", seed=None), id="gn_0"
),
pytest.param(
initializers.glorot_uniform(42), dict(class_name="glorot_uniform", seed=42), id="gu_0"
),
pytest.param(initializers.he_normal(), dict(class_name="he_normal", seed=None), id="hn_0"),
pytest.param(
initializers.he_uniform(42), dict(class_name="he_uniform", seed=42), id="hu_0"
),
pytest.param(
initializers.lecun_normal(), dict(class_name="lecun_normal", seed=None), id="ln_0"
),
pytest.param(
initializers.lecun_uniform(42), dict(class_name="lecun_uniform", seed=42), id="lu_0"
),
],
)
def test_keras_initializer_to_dict(initializer, initializer_dict):
assert get_concise_params_dict(keras_initializer_to_dict(initializer)) == initializer_dict
##################################################
# `get_concise_params_dict` Scenarios
##################################################
hh_arg_attrs = ["__hh_default_args", "__hh_default_kwargs", "__hh_used_args", "__hh_used_kwargs"]
empty_hh_args = [[], {}, [], {}]
_arg_dict = lambda _: dict(zip(hh_arg_attrs, _))
def build_model(self):
"""Build a critic (value) network that maps (state, action) pairs -> Q-values."""
# Define input layers
states = layers.Input(shape=(self.state_size, ), name='states')
actions = layers.Input(shape=(self.action_size, ), name='actions')
# Add hidden layer(s) for state pathway
net_states = layers.Dense(
units=400,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2))(states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation('relu')(net_states)
net_states = layers.Dense(
units=300,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2))(net_states)
net_states = layers.BatchNormalization()(net_states)
net_states = layers.Activation('relu')(net_states)
# Add hidden layer(s) for action pathway
net_actions = layers.Dense(
units=400,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2))(actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Activation('relu')(net_actions)
net_actions = layers.Dense(
units=300,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform(),
kernel_regularizer=regularizers.l2(self.L2))(net_actions)
net_actions = layers.BatchNormalization()(net_actions)
net_actions = layers.Activation('relu')(net_actions)
# Try different layer sizes, activations, add batch normalization, regularizers, etc.
# Combine state and action pathways
net = layers.Add()([net_states, net_actions])
net = layers.Activation('relu')(net)
# Add more layers to the combined network if needed
net = layers.Dense(units=200,
kernel_initializer=initializers.lecun_uniform(),
bias_initializer=initializers.lecun_uniform())(net)
net = layers.BatchNormalization()(net)
net = layers.Activation('relu')(net)
# Add final output layer to produce action values (Q values)
Q_values = layers.Dense(
units=1,
name='q_values',
kernel_initializer=initializers.RandomUniform(-0.003, 0.003),
bias_initializer=initializers.RandomUniform(-0.003, 0.003))(net)
# Create Keras model
self.model = models.Model(inputs=[states, actions], outputs=Q_values)
# Define optimizer and compile model for training with built-in loss function
optimizer = optimizers.Adam(lr=self.lr)
self.model.compile(optimizer=optimizer, loss='mse')
# Compute action gradients (derivative of Q values w.r.t. to actions)
action_gradients = K.gradients(Q_values, actions)
# Define an additional function to fetch action gradients (to be used by actor model)
self.get_action_gradients = K.function(
inputs=[*self.model.input, K.learning_phase()],
outputs=action_gradients)
def test_lecun_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
scale = np.sqrt(3. / fan_in)
_runner(initializers.lecun_uniform(), tensor_shape,
target_mean=0., target_max=scale, target_min=-scale)
def test_lecun_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
std = np.sqrt(1. / fan_in)
_runner(initializers.lecun_uniform(), tensor_shape,
target_mean=0., target_std=std)
pytest.param(initializers.glorot_normal(),
dict(class_name="glorot_normal", seed=None),
id="gn_0"),
pytest.param(initializers.glorot_uniform(42),
dict(class_name="glorot_uniform", seed=42),
id="gu_0"),
pytest.param(initializers.he_normal(),
dict(class_name="he_normal", seed=None),
id="hn_0"),
pytest.param(initializers.he_uniform(42),
dict(class_name="he_uniform", seed=42),
id="hu_0"),
pytest.param(initializers.lecun_normal(),
dict(class_name="lecun_normal", seed=None),
id="ln_0"),
pytest.param(initializers.lecun_uniform(42),
dict(class_name="lecun_uniform", seed=42),
id="lu_0"),
],
)
def test_keras_initializer_to_dict(initializer, initializer_dict):
assert get_concise_params_dict(
keras_initializer_to_dict(initializer)) == initializer_dict
##################################################
# `get_concise_params_dict` Scenarios
##################################################
hh_arg_attrs = [
"__hh_default_args", "__hh_default_kwargs", "__hh_used_args",
"__hh_used_kwargs"
y_, y_test = y[x_index], y[test_index]
validateSetSampler = StratifiedShuffleSplit(n_splits=10,
test_size=0.2,
train_size=0.8,
random_state=0)
for train_index, validate_index in testSetSampler.split(x_, y_):
x_train, x_val = x_[train_index], x_[validate_index]
y_train, y_val = y_[train_index], y_[validate_index]
# Model Template
model = Sequential() # declare model
model.add(
Dense(50,
input_shape=(28 * 28, ),
kernel_initializer=initializers.lecun_uniform(
seed=None))) # first layer
model.add(Activation('relu'))
model.add(Dense(30, activation='tanh'))
model.add(
Dense(100, activation='relu', use_bias=True, bias_initializer='zeros'))
model.add(
Dense(200, activation='relu', use_bias=True, bias_initializer='zeros'))
model.add(Dense(60, activation='tanh'))
model.add(
Dense(100, activation='relu', use_bias=True, bias_initializer='zeros'))
model.add(Dense(30, activation='tanh'))
model.add(Dense(10, kernel_initializer='he_normal')) # last layer
model.add(Activation('softmax'))
def test_lecun_uniform(tensor_shape):
fan_in, _ = initializers._compute_fans(tensor_shape)
scale = np.sqrt(3. / fan_in)
_runner(initializers.lecun_uniform(), tensor_shape,
target_mean=0., target_max=scale, target_min=-scale)
def Conv1DRegressorIn1(flag):
K.clear_session()
current_neighbor = space['neighbor']
current_idx_idx = space['idx_idx']
current_batch_size = space['batch_size']
current_dense_num = space['dense_num']
current_conv1D_filter_num1 = space['conv1D_filter_num1']
current_conv1D_filter_num2 = space['conv1D_filter_num2']
current_conv1D_filter_num3 = space['conv1D_filter_num3']
summary = True
verbose = 0
#
# setHyperParams
#
## hypers for data
neighbor = {{choice([50, 60, 70, 80, 90, 100, 110, 120, 130, 140])}}
idx_idx = {{choice([0,1,2,3,4,5,6,7,8])}}
idx_lst = [
[x for x in range(158) if x not in [24, 26]], # 去除无用特征
[x for x in range(158) if x not in [24, 26] + [x for x in range(1, 6)] + [x for x in range(16, 22)] + [40, 42]], # 去除无用特征+冗余特征
[x for x in range(158) if x not in [24, 26] + [x for x in range(0, 22)]], # 去除无用特征+方位特征
[x for x in range(158) if x not in [24, 26] + [22, 23, 26, 37, 38]], # 去除无用特征+深度特征
[x for x in range(158) if x not in [24, 26] + [x for x in range(27, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息
# [x for x in range(158) if x not in [24, 26] + [x for x in range(27, 34)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息1
# [x for x in range(158) if x not in [24, 26] + [x for x in range(34, 37)] + [x for x in range(40, 46)]],# 去除无用特征+二级结构信息2
[x for x in range(158) if x not in [24, 26] + [46, 47]], # 去除无用特征+实验条件
[x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(61, 81)] + [x for x in range(140, 155)]], # 去除无用特征+所有原子编码
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(48, 57)] + [x for x in range(140, 145)]],# 去除无用特征+原子编码1
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(61, 77)] + [x for x in range(145, 153)]],# 去除无用特征+原子编码2
# [x for x in range(158) if x not in [24, 26] + [39] + [x for x in range(57, 61)] + [x for x in range(77, 81)] + [x for x in range(153, 155)]],# 去除无用特征+原子编码3
[x for x in range(158) if x not in [24, 26] + [x for x in range(81, 98)]], # 去除无用特征+rosetta_energy
[x for x in range(158) if x not in [24, 26] + [x for x in range(98, 140)] + [x for x in range(155, 158)]]# 去除无用特征+msa
]
idx = idx_lst[idx_idx]
## hypers for net
lr = 1e-4 # 0.0001
batch_size = {{choice([1, 16, 32, 64])}}
epochs = 200
padding_style = 'same'
activator_Conv1D = 'elu'
activator_Dense = 'tanh'
dense_num = {{choice([64, 96, 128])}}
conv1D_filter_num1 = {{choice([16, 32])}}
conv1D_filter_num2 = {{choice([16, 32, 64])}}
conv1D_filter_num3 = {{choice([32,64])}}
dropout_rate_conv1D = 0.15
dropout_rate_dense = 0.25
initializer_Conv1D = initializers.lecun_uniform(seed=527)
initializer_Dense = initializers.he_normal(seed=527)
kernel_size = 5
l2_rate = 0.001
loss_type = logcosh
metrics = ('mae', pearson_r, rmse)
pool_size = 2
def _data(fold_num, neighbor, idx):
train_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_train_center_CA_PCA_False_neighbor_140.npz' % fold_num
val_data_pth = '/dl/sry/mCNN/dataset/deepddg/npz/wild/cross_valid/cro_fold%s_valid_center_CA_PCA_False_neighbor_140.npz' % fold_num
## train data
train_data = np.load(train_data_pth)
x_train = train_data['x']
y_train = train_data['y']
ddg_train = train_data['ddg'].reshape(-1)
## select kneighbor atoms
x_train_kneighbor_lst = []
for sample in x_train:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_train_kneighbor_lst.append(sample[indices, :])
x_train = np.array(x_train_kneighbor_lst)
## idx
x_train = x_train[:, :, idx]
## val data
val_data = np.load(val_data_pth)
x_val = val_data['x']
y_val = val_data['y']
ddg_val = val_data['ddg'].reshape(-1)
## select kneighbor atoms
x_val_kneighbor_lst = []
for sample in x_val:
dist_arr = sample[:, 0]
indices = sorted(dist_arr.argsort()[:neighbor])
x_val_kneighbor_lst.append(sample[indices, :])
x_val = np.array(x_val_kneighbor_lst)
## idx
x_val = x_val[:, :, idx]
# sort row default is chain, pass
# reshape and one-hot
y_train = to_categorical(y_train)
y_val = to_categorical(y_val)
# normalization
train_shape = x_train.shape
val_shape = x_val.shape
col_train = train_shape[-1]
col_val = val_shape[-1]
x_train = x_train.reshape((-1, col_train))
x_val = x_val.reshape((-1, col_val))
mean = x_train.mean(axis=0)
std = x_train.std(axis=0)
std[np.argwhere(std == 0)] = 0.01
x_train -= mean
x_train /= std
x_val -= mean
x_val /= std
x_train = x_train.reshape(train_shape)
x_val = x_val.reshape(val_shape)
print('x_train: %s'
'\ny_train: %s'
'\nddg_train: %s'
'\nx_val: %s'
'\ny_val: %s'
'\nddg_val: %s'
% (x_train.shape, y_train.shape, ddg_train.shape,
x_val.shape, y_val.shape, ddg_val.shape))
return x_train, y_train, ddg_train, x_val, y_val, ddg_val
#
# cross_valid
#
hyper_param_tag = '%s_%s_%s_%s_%s_%s_%s' % (
current_neighbor, current_idx_idx, current_batch_size, current_dense_num,
current_conv1D_filter_num1, current_conv1D_filter_num2, current_conv1D_filter_num3)
modeldir = '/dl/sry/projects/from_hp/mCNN/src/Network/deepddg/opt_all_simpleNet/model/%s-%s' % (
hyper_param_tag, time.strftime("%Y.%m.%d.%H.%M.%S", time.localtime()))
os.makedirs(modeldir, exist_ok=True)
opt_lst = []
for k_count in range(1,11):
print('\n** fold %s is processing **\n' % k_count)
filepth = '%s/fold_%s_weights-best.h5' % (modeldir, k_count)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.33,
patience=5,
verbose=verbose,
mode='min',
min_lr=1e-8,
),
callbacks.EarlyStopping(
monitor='val_loss',
patience=10,
verbose=verbose
),
callbacks.ModelCheckpoint(
filepath=filepth,
monitor='val_mean_absolute_error',
verbose=verbose,
save_best_only=True,
mode='min',
save_weights_only=True)
]
x_train, y_train, ddg_train, x_val, y_val, ddg_val = _data(k_count,neighbor,idx)
row_num, col_num = x_train.shape[1:3]
#
# build net
#
network = models.Sequential()
network.add(layers.Conv1D(filters=conv1D_filter_num1,
kernel_size=kernel_size,
kernel_initializer=initializer_Conv1D,
kernel_regularizer=regularizers.l2(l2_rate),
activation=activator_Conv1D,
input_shape=(row_num, col_num)))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.MaxPooling1D(pool_size=pool_size,
padding=padding_style))
network.add(layers.SeparableConv1D(filters=conv1D_filter_num2,
kernel_size=kernel_size,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate),
activation=activator_Conv1D))
network.add(layers.Dropout(dropout_rate_conv1D))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.MaxPooling1D(pool_size=pool_size,
padding=padding_style))
network.add(layers.SeparableConv1D(filters=conv1D_filter_num3,
kernel_size=3,
depthwise_initializer=initializer_Conv1D,
pointwise_initializer=initializer_Conv1D,
depthwise_regularizer=regularizers.l2(l2_rate),
pointwise_regularizer=regularizers.l2(l2_rate),
activation=activator_Conv1D))
network.add(layers.Dropout(dropout_rate_conv1D))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.MaxPooling1D(pool_size=pool_size,
padding=padding_style))
network.add(layers.Flatten())
network.add(layers.Dense(dense_num,
kernel_initializer=initializer_Dense,
kernel_regularizer=regularizers.l2(l2_rate),
activation=activator_Dense))
network.add(layers.Dropout(dropout_rate_dense))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.Dense(1))
if summary:
trainable_count = int(
np.sum([K.count_params(p) for p in set(network.trainable_weights)]))
non_trainable_count = int(
np.sum([K.count_params(p) for p in set(network.non_trainable_weights)]))
print('Total params: {:,}'.format(trainable_count + non_trainable_count))
print('Trainable params: {:,}'.format(trainable_count))
print('Non-trainable params: {:,}'.format(non_trainable_count))
# print(network.summary())
# rmsp = optimizers.RMSprop(lr=0.0001)
adam = optimizers.Adam(lr=lr)
network.compile(optimizer=adam, # 'rmsprop', # SGD,adam,rmsprop
loss=loss_type,
metrics=list(metrics)) # mae平均绝对误差（mean absolute error） accuracy
result = network.fit(x=x_train,
y=ddg_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=(x_val, ddg_val),
shuffle=True,
)
# print('\n----------History:\n%s'%result.history)
#
# save
#
save_train_cv(network, modeldir, result.history,k_count)
opt_lst.append(np.mean(result.history['val_mean_absolute_error'][-10:]))
opt_loss = np.mean(opt_lst)
#
# print hyper combination group and current loss value
#
print('\n@current_hyper_tag: %s'
'\n@current optmized_loss: %s'
%(hyper_param_tag, opt_loss))
# return {'loss': validation_loss, 'status': STATUS_OK, 'model':model}
return {'loss': opt_loss, 'status': STATUS_OK}
def buildModel(self):
"""Model layers"""
# character input
character_input = Input(shape=(None, 52,), name="Character_input")
embed_char_out = TimeDistributed(
# Embedding(len(self.char2Idx), 30, embeddings_initializer=RandomUniform(minval=-0.5, maxval=0.5)), name="Character_embedding")(character_input)
Embedding(len(self.char2Idx), 30,
embeddings_initializer=lecun_uniform(seed=None)
), name="Character_embedding")(character_input) # Xavier Normal Initializer
dropout = Dropout(self.dropout)(embed_char_out)
# CNN
conv1d_out = TimeDistributed(Conv1D(kernel_size=self.conv_size, filters=30, padding='same', activation='tanh', strides=1), name="Convolution")(dropout)
maxpool_out = TimeDistributed(MaxPooling1D(52), name="Maxpool")(conv1d_out)
char = TimeDistributed(Flatten(), name="Flatten")(maxpool_out)
char = Dropout(self.dropout)(char)
# word-level input
words_input = Input(shape=(None,), dtype='int32', name='words_input')
words = Embedding(input_dim=self.wordEmbeddings.shape[0], output_dim=self.wordEmbeddings.shape[1], weights=[self.wordEmbeddings],
trainable=False)(words_input)
# case-info input
casing_input = Input(shape=(None,), dtype='int32', name='casing_input')
casing = Embedding(output_dim=self.caseEmbeddings.shape[1], input_dim=self.caseEmbeddings.shape[0], weights=[self.caseEmbeddings],
trainable=False)(casing_input)
# concat & BLSTM
output = concatenate([words, casing, char])
output = Bidirectional(LSTM(self.lstm_state_size,
return_sequences=True,
dropout=self.dropout, # on input to each LSTM block
recurrent_dropout=self.dropout_recurrent # on recurrent input signal
), name="BLSTM")(output)
# words_output = Bidirectional(LSTM(self.lstm_state_size,
# return_sequences=True,
# dropout=self.dropout, # on input to each LSTM block
# recurrent_dropout=self.dropout_recurrent # on recurrent input signal
# ), name="words_BLSTM")(words)
# casing_output = Bidirectional(LSTM(self.lstm_state_size,
# return_sequences=True,
# dropout=self.dropout, # on input to each LSTM block
# recurrent_dropout=self.dropout_recurrent # on recurrent input signal
# ), name="casing_BLSTM")(casing)
# char_output = Bidirectional(LSTM(self.lstm_state_size,
# return_sequences=True,
# dropout=self.dropout, # on input to each LSTM block
# recurrent_dropout=self.dropout_recurrent # on recurrent input signal
# ), name="char_BLSTM")(char)
# output = concatenate([words_output, casing_output, char_output])
output = TimeDistributed(Dense(len(self.label2Idx), activation='softmax'),name="Softmax_layer")(output)
# set up model
self.model = Model(inputs=[words_input, casing_input, character_input], outputs=[output])
self.model.compile(loss='sparse_categorical_crossentropy', optimizer=self.optimizer)
self.init_weights = self.model.get_weights()
plot_model(self.model, to_file='model.png')
print("Model built. Saved model.png\n")
def TrainResNet(x_train,
y_train,
x_val=None,
y_val=None,
class_weights_dict=None,
filepth=None,
epochs=200,
lr=1e-2,
verbose=1):
summary = True
batch_size = 128
optimizer = 'adam'
activator = 'relu'
kernel_size = 3
pool_size = 2
init_Conv1D = initializers.lecun_uniform()
init_Dense = initializers.he_normal()
padding_style = 'same'
dropout_rate = 0.025
l2_coeff = 1e-3
loss_type = 'categorical_crossentropy'
metrics = ('acc', )
## used in the dilation loop
dilation_lower = 1
dilation_upper = 16
dilation1D_layers = 16
dilation1D_filter_num = 16
## used in the reduce loop
residual_stride = 2
reduce_layers = 6 # 100 -> 50 -> 25 -> 13 -> 7 -> 4 -> 2
reduce1D_filter_num = 16
dense_num = 128
dropout_dense = 0.25
if lr > 0:
if optimizer == 'adam':
chosed_optimizer = optimizers.Adam(lr=lr)
elif optimizer == 'sgd':
chosed_optimizer = optimizers.SGD(lr=lr)
elif optimizer == 'rmsprop':
chosed_optimizer = optimizers.RMSprop(lr=lr)
if x_val is None or y_val is None:
val_data = None
my_callbacks = None
else:
val_data = (x_val, y_val)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.3,
patience=5,
verbose=verbose,
),
callbacks.EarlyStopping(
monitor='val_acc',
min_delta=1e-4,
patience=20,
mode='max',
verbose=verbose,
),
]
if filepth is not None:
my_callbacks += [
callbacks.ModelCheckpoint(
filepath=filepth,
monitor='val_acc',
mode='max',
save_best_only=True,
save_weights_only=True,
verbose=verbose,
)
]
#
# build
#
## basic Conv1D
input_layer = Input(shape=x_train.shape[1:])
y = layers.SeparableConv1D(filters=dilation1D_filter_num,
kernel_size=1,
padding=padding_style,
kernel_initializer=init_Conv1D,
activation=activator)(input_layer)
res = layers.BatchNormalization(axis=-1)(y)
## loop with Conv1D with dilation (padding='same')
for _ in range(dilation1D_layers):
y = layers.SeparableConv1D(
filters=dilation1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
kernel_initializer=init_Conv1D,
activation=activator,
kernel_regularizer=regularizers.l2(l2_coeff))(res)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_rate)(y)
y = layers.SeparableConv1D(
filters=dilation1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
dilation_rate=dilation_lower,
kernel_initializer=init_Conv1D,
activation=activator,
kernel_regularizer=regularizers.l2(l2_coeff))(y)
y = layers.BatchNormalization(axis=-1)(y)
res = layers.add([y, res])
dilation_lower *= 2
if dilation_lower > dilation_upper:
dilation_lower = 1
## residual block to reduce dimention.
for _ in range(reduce_layers):
y = layers.SeparableConv1D(
filters=reduce1D_filter_num,
kernel_size=kernel_size,
padding=padding_style,
kernel_initializer=init_Conv1D,
activation=activator,
kernel_regularizer=regularizers.l2(l2_coeff))(res)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_rate)(y)
y = layers.MaxPooling1D(pool_size, padding=padding_style)(y)
res = layers.SeparableConv1D(
filters=reduce1D_filter_num,
kernel_size=kernel_size,
strides=residual_stride,
padding=padding_style,
kernel_initializer=init_Conv1D,
activation=activator,
kernel_regularizer=regularizers.l2(l2_coeff))(res)
res = layers.add([y, res])
## flat & dense
y = layers.Flatten()(y)
y = layers.Dense(dense_num, activation=activator)(y)
y = layers.BatchNormalization(axis=-1)(y)
y = layers.Dropout(dropout_dense)(y)
output_layer = layers.Dense(10, activation='softmax')(y)
model = models.Model(inputs=input_layer, outputs=output_layer)
if summary:
model.summary()
model.compile(
optimizer=chosed_optimizer,
loss=loss_type,
metrics=list(metrics) # accuracy
)
# K.set_session(tf.Session(graph=model.output.graph))
# init = K.tf.global_variables_initializer()
# K.get_session().run(init)
result = model.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=val_data,
shuffle=True,
class_weight=class_weights_dict)
return model, result.history
def TrainConv1D(x_train,
y_train,
x_val=None,
y_val=None,
class_weights_dict=None,
filepth=None,
epochs=200,
lr=1e-2,
verbose=1):
summary = False
batch_size = 128
optimizer = 'adam'
activator = 'relu'
pool_size = 2
init_Conv1D = initializers.lecun_uniform()
init_Dense = initializers.he_normal()
padding_style = 'same'
drop_rate = 0.025
l2_coeff = 1e-3
loss_type = 'categorical_crossentropy'
metrics = ('acc', )
loop_conv_num = 4 # 100 -> 50 -> 25 -> 13 -> 7 -> 4
dense_num = 128
dropout_dense = 0.25
if lr > 0:
if optimizer == 'adam':
chosed_optimizer = optimizers.Adam(lr=lr)
elif optimizer == 'sgd':
chosed_optimizer = optimizers.SGD(lr=lr)
elif optimizer == 'rmsprop':
chosed_optimizer = optimizers.RMSprop(lr=lr)
if x_val is None or y_val is None:
val_data = None
my_callbacks = None
else:
val_data = (x_val, y_val)
my_callbacks = [
callbacks.ReduceLROnPlateau(
monitor='val_loss',
factor=0.3,
patience=5,
verbose=verbose,
),
callbacks.EarlyStopping(
monitor='val_acc',
min_delta=1e-4,
patience=20,
mode='max',
verbose=verbose,
),
]
if filepth is not None:
my_callbacks += [
callbacks.ModelCheckpoint(
filepath=filepth,
monitor='val_acc',
mode='max',
save_best_only=True,
save_weights_only=True,
verbose=verbose,
)
]
#
# build model
#
network = models.Sequential()
network.add(
layers.SeparableConv1D(filters=16,
kernel_size=5,
activation=activator,
padding=padding_style,
depthwise_initializer=init_Conv1D,
pointwise_initializer=init_Conv1D,
depthwise_regularizer=regularizers.l2(l2_coeff),
pointwise_regularizer=regularizers.l1(l2_coeff),
input_shape=(x_train.shape[1:])))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.Dropout(drop_rate))
network.add(layers.MaxPooling1D(pool_size=pool_size,
padding=padding_style))
for _ in range(loop_conv_num):
network.add(
layers.SeparableConv1D(
filters=32,
kernel_size=5,
activation=activator,
padding=padding_style,
depthwise_initializer=init_Conv1D,
pointwise_initializer=init_Conv1D,
depthwise_regularizer=regularizers.l2(l2_coeff),
pointwise_regularizer=regularizers.l1(l2_coeff),
))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.Dropout(drop_rate))
network.add(
layers.MaxPooling1D(pool_size=pool_size, padding=padding_style))
network.add(
layers.SeparableConv1D(
filters=64,
kernel_size=3,
activation=activator,
padding=padding_style,
depthwise_initializer=init_Conv1D,
pointwise_initializer=init_Conv1D,
depthwise_regularizer=regularizers.l2(l2_coeff),
pointwise_regularizer=regularizers.l1(l2_coeff),
))
network.add(layers.BatchNormalization(axis=-1))
network.add(layers.Dropout(drop_rate))
network.add(layers.MaxPooling1D(pool_size=pool_size,
padding=padding_style))
network.add(layers.Flatten())
network.add(
layers.Dense(units=dense_num,
kernel_initializer=init_Dense,
activation=activator))
network.add(layers.Dropout(dropout_dense))
network.add(
layers.Dense(units=10,
kernel_initializer=init_Dense,
activation='softmax'))
if summary:
print(network.summary())
network.compile(optimizer=chosed_optimizer,
loss=loss_type,
metrics=list(metrics))
result = network.fit(x=x_train,
y=y_train,
batch_size=batch_size,
epochs=epochs,
verbose=verbose,
callbacks=my_callbacks,
validation_data=val_data,
shuffle=True,
class_weight=class_weights_dict)
return network, result.history
model = Sequential()
model.add(
Dense(count,
input_dim=input_dim,
kernel_initializer=wi.lecun_normal(seed=seed),
bias_initializer=wi.lecun_normal(seed=seed)))
plot_weights(weights=model.get_weights(),
x=np.arange(0, count, 1),
title='lecun_normal')
model = Sequential()
model.add(
Dense(count,
input_dim=input_dim,
kernel_initializer=wi.lecun_uniform(seed=seed),
bias_initializer=wi.lecun_uniform(seed=seed)))
plot_weights(weights=model.get_weights(),
x=np.arange(0, count, 1),
title='lecun_uniform')
model = Sequential()
model.add(
Dense(count,
input_dim=input_dim,
kernel_initializer=wi.glorot_normal(seed=seed),
bias_initializer=wi.glorot_normal(seed=seed)))
plot_weights(weights=model.get_weights(),
x=np.arange(0, count, 1),
title='glorot_normal')
factor_names.extend(size_name)
factor_names.extend(momentum_name)
factor_names.extend(quality_name)
factor_names.extend(volatility_name)
factor_name = ''.join(factor_names)
if factor_name:
_data_sets[factor_name[:-1]] = (get_data,
factor_name[:-1])
data_sets = LazyDict(_data_sets)
activations = {LINEAR: linear, TAHN: tanh, RELU: relu}
initializers = {
LECUN_NORMAL: lecun_normal(),
LECUN_UNIFORM: lecun_uniform(),
HE_NORMAL: he_normal(),
HE_UNIFORM: he_uniform(),
GLOROT_NORMAL: glorot_normal(),
GLOROT_UNIFORM: glorot_uniform(),
ZEROS: zeros()
}
regularizers = {NONE: None, L1: l1(), L2: l2(), L1_L2: l1_l2()}
hidden_layers = {
NN3_1: [70],
NN3_2: [80],
NN3_3: [100],
NN3_4: [120],
DNN5_1: [100, 50, 10],
