class NN:
"""
The NN class wraps a keras Sequential model to reduce the interface methods
Notice:
Difference to dqn is just the setter and getter methods for the weights
"""
def __init__(self, env, atoms, alpha: float = 0.001, decay: float = 0.0001):
"""
We initialize our functional model, therefore we need Input Shape and Output Shape
:param env:
:param alpha:
:param decay:
"""
self.alpha = alpha
self.decay = decay
self.model = None
self.atoms = atoms
# new to D-DDQN
self.init_model(env.observation_space.shape[0], env.action_space.n)
def init_model(self, input_shape: int, n_actions: int):
"""
Initializing our keras sequential model
:return: initialized model
"""
input = Input(shape=(input_shape,))
h1 = Dense(64, activation='relu')(input)
h2 = Dense(64, activation='relu')(h1)
outputs = []
for _ in range(n_actions):
outputs.append(Dense(self.atoms, activation='softmax')(h2))
self.model = Model(input, outputs)
def predict(self, *args, **kwargs):
"""
By wrapping the keras predict method we can handle our net as a standalone object
:param args: interface to keras.model.predict
:return: prediction
"""
return self.model.predict(*args, **kwargs)
def fit(self, *args, **kwargs):
"""
By wrapping the keras fit method we can handle our net as a standalone object
:param args: interface to keras.model.fit
:return: history object
"""
return self.model.fit(*args, **kwargs)
def get_weights(self):
"""
Passing the arguments to keras get_weights
"""
return self.model.get_weights()
def set_weights(self, *args, **kwargs):
"""
Passing the arguments to keras set_weights
"""
self.model.set_weights(*args, *kwargs)
class PolicyModel:
def __init__(self, config: Config):
self.config = config
self.model: Model = None
def load_model(self):
try:
self.model = load_model(str(self.model_file_path))
logger.info(f"loading policy model success")
return True
except Exception:
return False
def save_model(self):
logger.info(f"saving policy model")
self.model_file_path.parent.mkdir(parents=True, exist_ok=True)
self.model.save(str(self.model_file_path), include_optimizer=False)
def build(self):
logger.info(f"setup state model")
in_state = Input((self.config.model.vae.latent_dim, ), name="in_state")
in_keys = Input((1, ), name="in_keys")
in_time = Input((1, ), name="in_time")
in_actions = Input((self.config.policy_model.n_actions, ),
name="in_actions")
in_rarity = Input((1, ), name="in_rarity")
in_all = Concatenate(name="in_all")(
[in_state, in_keys, in_time, in_actions, in_rarity])
x = Dense(self.config.policy_model.hidden_size,
activation="tanh",
name="hidden",
kernel_regularizer=l2(0.0001))(in_all)
out_actions = Dense(self.config.policy_model.n_actions,
activation="softmax",
name="parameters",
kernel_regularizer=l2(0.0001))(x)
out_keep_rate = Dense(1, activation="sigmoid",
name="keep_rate")(in_all)
self.model = Model([in_state, in_keys, in_time, in_actions, in_rarity],
[out_actions, out_keep_rate],
name="policy_model")
def predict(self, state, keys, time_remain, in_actions, in_rarity):
actions, kr = self.model.predict([
np.expand_dims(state, axis=0),
np.array([[keys]]),
np.array([[time_remain]]),
np.expand_dims(in_actions, axis=0),
np.array([[in_rarity]]),
])
return actions[0], kr[0]
def get_parameters(self):
return self.model.get_weights()
def set_parameters(self, parameters):
self.model.set_weights(parameters)
def compile(self):
self.model.compile(
optimizer=Adam(lr=0.00001),
loss=[kullback_leibler_divergence, mean_squared_error],
loss_weights=[1., 0.])
@property
def model_file_path(self):
return self.config.resource.model_dir / "policy_weights.h5"
class NN:
"""
The biggest change for the duelling DQN is, that we must define a more complex DQN architecture
The architecture must define our Value and Advantage Layers
The way we define our model:
<<<>>>
The Keras functional API is a way to create models that are more flexible than the tf.keras.Sequential API.
The functional API can handle models with NON-LINEAR topology, SHARED layers, and even MULTIPLE inputs or outputs.
Read more here: https://keras.io/guides/functional_api/
<<<>>>
"""
def __init__(self, env, alpha: float = 0.001, decay: float = 0.0001):
"""
We initialize our functional model, therefore we need Input Shape and Output Shape
:param env:
:param alpha:
:param decay:
"""
self.alpha = alpha
self.decay = decay
self.model = None
# new to D-DDQN
self.init_model(env.observation_space.shape[0], env.action_space.n)
def init_model(self, input_shape: int, n_actions: int):
inp = Input(shape=(input_shape, ))
layer_shared1 = Dense(64, activation='relu')(inp)
layer_shared1 = BatchNormalization()(layer_shared1)
layer_shared2 = Dense(64, activation='relu')(layer_shared1)
layer_shared2 = BatchNormalization()(layer_shared2)
layer_v1 = Dense(64, activation='relu')(layer_shared2)
layer_v1 = BatchNormalization()(layer_v1)
layer_a1 = Dense(64, activation='relu')(layer_shared2)
layer_a1 = BatchNormalization()(layer_a1)
# the value layer ouput is a scalar value
layer_v2 = Dense(1, activation='linear')(layer_v1)
# The advantage function subtracts the value of the state from the Q
# function to obtain a relative measure of the importance of each action.
layer_a2 = Dense(n_actions, activation='linear')(layer_a1)
# the q layer combines the two streams of value and advantage function
# the lambda functional layer can perform lambda expressions on keras layers
# read more here : https://keras.io/api/layers/core_layers/lambda/
# the lambda equation is defined in https://arxiv.org/pdf/1511.06581.pdf on equation (9)
layer_q = Lambda(lambda x: x[0][:] + x[1][:] - K.mean(x[1][:]),
output_shape=(n_actions, ))([layer_v2, layer_a2])
self.model = Model(inp, layer_q)
self.model.compile(optimizer=Adam(lr=self.alpha), loss='mse')
def predict(self, *args, **kwargs):
"""
By wrapping the keras predict method we can handle our net as a standalone object
:param args: interface to keras.model.predict
:return: prediction
"""
return self.model.predict(*args, **kwargs)
def fit(self, *args, **kwargs):
"""
By wrapping the keras fit method we can handle our net as a standalone object
:param args: interface to keras.model.fit
:return: history object
"""
return self.model.fit(*args, **kwargs)
def get_weights(self):
"""
Passing the arguments to keras get_weights
"""
return self.model.get_weights()
def set_weights(self, *args, **kwargs):
"""
Passing the arguments to keras set_weights
"""
self.model.set_weights(*args, *kwargs)
# name= 'biInteraction')([atom_embedding, protSeq_embedding, atom_split, protSeq_len])
affinity = ConcatMlp(hidden_list=args.biInteraction_hidden,
dropout=args.dropout,
activation='tanh',
weight_decay=args.weight_decay)(
[atom_embedding, protSeq_embedding, atom_split])
# DrugPropertyModel = Model(inputs= mol_input, outputs= mol_property, name= 'drugPropertyModel')
DTAModel = Model(inputs=[mol_input, protSeq_input],
outputs=affinity,
name="DTAmodel")
init_weight_subdir = chkpt_dir + '/initial/'
if not os.path.exists(init_weight_subdir):
os.makedirs(init_weight_subdir)
# DTAModel.save_weights(init_weight_subdir)
init_weight = DTAModel.get_weights()
if args.pretrain:
print("pretrain...")
DrugPropertyModel.summary()
pretrain(
DrugPropertyModel,
dataset="kiba_origin",
dir_prefix=prefix,
epoches=args.pretrain_epoches,
batchsize=64,
lr=1E-4,
patience=8,
)
print("pretrain conclude.")
