class nfp(object): """NFP manager This class has the generator function of NFP and updator of NN for learning the generator of NFP. Args: d: Dimension of NFP. f: Dimension of the feature for generating NFP. R: Radius for generating NFP. gpu (boolean): GPU flag. If you want to use GPU, set it True. """ def __init__(self, d, f, R, gpu): self.d = d self.f = f self.R = R self.gpu = gpu g = ChainList(*[L.Linear(1, f) for i in six.moves.range(AtomIdMax)]) H = ChainList(*[L.Linear(f, f) for i in six.moves.range(R)]) W = ChainList(*[L.Linear(f, d) for i in six.moves.range(R + 1)]) self.optimizer = optimizers.Adam() self.model = Chain(H=H, W=W, g=g) if gpu: self.model.to_gpu(0) self.optimizer.setup(self.model) self.to = [[] for i in six.moves.range(2)] self.atom_sid = [[] for i in six.moves.range(2)] self.anum = [[] for i in six.moves.range(2)] def get_nfp(self, sids, train=True): """Generates NFP. Args: sids (int[]): List of substance IDs. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: fp: Dictionary of NFPs. Key is a substance ID. """ d, f, R = self.d, self.f, self.R def add_var(x): if self.gpu: return Variable(cuda.to_gpu(x, 0), volatile=not train) else: return Variable(x, volatile=not train) if train: ti = 0 else: ti = 1 to = self.to[ti] atom_sid = self.atom_sid[ti] anum = self.anum[ti] if len(to) == 0: for sid in sids: mol = data.load_sdf(sid) atoms = mol.GetAtoms() n = len(atoms) base = len(to) to += [[] for i in six.moves.range(n)] atom_sid += [sid for i in six.moves.range(n)] anum += [1 for i in six.moves.range(n)] for atom in atoms: anum[base + atom.GetIdx()] = atom.GetAtomicNum() to[base + atom.GetIdx()] = [ base + n_atom.GetIdx() for n_atom in atom.GetNeighbors() ] for i in six.moves.range(len(to)): if len(to[i]) == 0: to[i].append(i) V = len(atom_sid) vec = [[] for i in six.moves.range(R + 1)] fp = {} for l in six.moves.range(R + 1): vec[l] = [ add_var(np.zeros([1, f], dtype='float32')) for i in six.moves.range(V) ] for sid in sids: fp[sid] = add_var(np.zeros([1, d], dtype='float32')) for i in six.moves.range(V): vec[0][i] += self.model.g[anum[i]](add_var( np.array([[1]], dtype='float32'))) p = [[] for i in six.moves.range(R)] for l in six.moves.range(R): p[l] = [ to[i][np.random.randint(len(to[i]))] for i in six.moves.range(V) ] for i in six.moves.range(V): vec[l + 1][i] = F.tanh(self.model.H[l](vec[l][i] + vec[l][p[l][i]])) tmp = [[] for i in six.moves.range(R + 1)] for l in six.moves.range(R + 1): for i in six.moves.range(V): tmp[l].append(F.softmax(self.model.W[l](vec[l][i]))) for l in six.moves.range(R + 1): for i in six.moves.range(V): fp[atom_sid[i]] += tmp[l][i] return fp def update(self, sids, y, net, train=True): """Updates NFP NN. Args: sids (int[]): Substance ID. y (np.array(int32[])[2]): Activity data. y[0] is for the training dataset and y[1] is for the test dataset. net (nn.NN): Classifier of QSAR. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: result (float): Overall accuracy on the test dataset. """ self.model.zerograds() fps = self.get_nfp(sids[0] + sids[1], train) x_train = [fps[sid] for sid in sids[0]] x_test = [fps[sid] for sid in sids[1]] for x in x_train: x.volatile = 'off' for x in x_test: x.volatile = 'off' result = net.train(x_train, y[0], x_test, y[1], train, self.gpu) self.optimizer.update() return result
class nfp(object): """NFP manager This class has the generator function of NFP and updator of NN for learning the generator of NFP. Args: d: Dimension of NFP. f: Dimension of the feature for generating NFP. R: Radius for generating NFP. """ def __init__(self, d, f, R): self.d = d self.f = f self.R = R g = ChainList(*[L.Linear(1, f) for i in six.moves.range(AtomIdMax)]) H = ChainList(*[ChainList(*[L.Linear(f, f) for i in six.moves.range(R)]) for j in six.moves.range(5)]) W = ChainList(*[L.Linear(f, d) for i in six.moves.range(R)]) self.model = Chain(H=H, W=W, g=g) self.optimizer = optimizers.Adam() self.optimizer.setup(self.model) def get_nfp(self, sid, train=True): """Generates NFP. Args: sid (int): Substance ID. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: fp: NFP. """ d, f, R = self.d, self.f, self.R mol = data.load_sdf(sid) atoms = mol.GetAtoms() n = len(atoms) fp = Variable(np.zeros([1, d], dtype='float32'), volatile=not train) r = [[Variable(np.zeros([1, f], dtype='float32'), volatile=not train) for i in six.moves.range(n)] for j in six.moves.range(R + 1)] for atom in atoms: a = atom.GetIdx() anum = atom.GetAtomicNum() r[0][a] += self.model.g[anum](Variable(np.array([[1]], dtype='float32'), volatile=not train)) for l in six.moves.range(R): v = [Variable(np.zeros([1, f], dtype='float32'), volatile=not train) for i in six.moves.range(n)] for atom in atoms: a = atom.GetIdx() v[a] += r[l][a] for n_atom in atom.GetNeighbors(): na = n_atom.GetIdx() v[a] += r[l][na] for atom in atoms: a = atom.GetIdx() deg = atom.GetDegree() deg = min(5, max(1, deg)) r[l + 1][a] = F.tanh(self.model.H[deg - 1][l](v[a])) i = F.softmax(self.model.W[l](r[l + 1][a])) fp += i return fp def update(self, sids, y, net, train=True): """Updates NFP NN. Args: sids (int[]): Substance ID. y (np.array(int32[])[2]): Activity data. y[0] is for the training dataset and y[1] is for the test dataset. net (nn.NN): Classifier of QSAR. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: result (float): Overall accuracy on the test dataset. """ def get_nfps(sids, train=True): print('generate fingerprints...') fps = {} for i, sid in enumerate(sids[0] + sids[1]): fps[sid] = self.get_nfp(sid, train) print('done.') return fps self.model.zerograds() fps = get_nfps(sids, train) x_train = [fps[sid] for sid in sids[0]] x_test = [fps[sid] for sid in sids[1]] for x in x_train: x.volatile = 'off' for x in x_test: x.volatile = 'off' result = net.train(x_train, y[0], x_test, y[1], train) self.optimizer.update() return result
class nfp(object): """NFP manager This class has the generator function of NFP and updator of NN for learning the generator of NFP. Args: d: Dimension of NFP. f: Dimension of the feature for generating NFP. R: Radius for generating NFP. gpu (boolean): GPU flag. If you want to use GPU, set it True. """ def __init__(self, d, f, R, gpu): self.d = d self.f = f self.R = R self.gpu = gpu g = ChainList(*[L.Linear(1, f) for i in six.moves.range(AtomIdMax)]) H = ChainList(*[L.Linear(f, f) for i in six.moves.range(R)]) W = ChainList(*[L.Linear(f, d) for i in six.moves.range(R + 1)]) self.optimizer = optimizers.Adam() self.model = Chain(H=H, W=W, g=g) if gpu: self.model.to_gpu(0) self.optimizer.setup(self.model) self.to = [[] for i in six.moves.range(2)] self.atom_sid = [[] for i in six.moves.range(2)] self.anum = [[] for i in six.moves.range(2)] def get_nfp(self, sids, train=True): """Generates NFP. Args: sids (int[]): List of substance IDs. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: fp: Dictionary of NFPs. Key is a substance ID. """ d, f, R = self.d, self.f, self.R def add_var(x): if self.gpu: return Variable(cuda.to_gpu(x, 0), volatile=not train) else: return Variable(x, volatile=not train) if train: ti = 0 else: ti = 1 to = self.to[ti] atom_sid = self.atom_sid[ti] anum = self.anum[ti] if len(to) == 0: for sid in sids: mol = data.load_sdf(sid) atoms = mol.GetAtoms() n = len(atoms) base = len(to) to += [[] for i in six.moves.range(n)] atom_sid += [sid for i in six.moves.range(n)] anum += [1 for i in six.moves.range(n)] for atom in atoms: anum[base + atom.GetIdx()] = atom.GetAtomicNum() to[base + atom.GetIdx()] = [base + n_atom.GetIdx() for n_atom in atom.GetNeighbors()] for i in six.moves.range(len(to)): if len(to[i]) == 0: to[i].append(i) V = len(atom_sid) vec = [[] for i in six.moves.range(R + 1)] fp = {} for l in six.moves.range(R + 1): vec[l] = [add_var(np.zeros([1, f], dtype='float32')) for i in six.moves.range(V)] for sid in sids: fp[sid] = add_var(np.zeros([1, d], dtype='float32')) for i in six.moves.range(V): vec[0][i] += self.model.g[anum[i]](add_var(np.array([[1]], dtype='float32'))) p = [[] for i in six.moves.range(R)] for l in six.moves.range(R): p[l] = [to[i][np.random.randint(len(to[i]))] for i in six.moves.range(V)] for i in six.moves.range(V): vec[l + 1][i] = F.tanh(self.model.H[l] (vec[l][i] + vec[l][p[l][i]])) tmp = [[] for i in six.moves.range(R + 1)] for l in six.moves.range(R + 1): for i in six.moves.range(V): tmp[l].append(F.softmax(self.model.W[l](vec[l][i]))) for l in six.moves.range(R + 1): for i in six.moves.range(V): fp[atom_sid[i]] += tmp[l][i] return fp def update(self, sids, y, net, train=True): """Updates NFP NN. Args: sids (int[]): Substance ID. y (np.array(int32[])[2]): Activity data. y[0] is for the training dataset and y[1] is for the test dataset. net (nn.NN): Classifier of QSAR. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: result (float): Overall accuracy on the test dataset. """ self.model.zerograds() fps = self.get_nfp(sids[0] + sids[1], train) x_train = [fps[sid] for sid in sids[0]] x_test = [fps[sid] for sid in sids[1]] for x in x_train: x.volatile = 'off' for x in x_test: x.volatile = 'off' result = net.train(x_train, y[0], x_test, y[1], train, self.gpu) self.optimizer.update() return result
class QNet: # Hyper-Parameters gamma = 0.95 # Discount factor timestep_per_episode = 5000 initial_exploration = timestep_per_episode * 1 # Initial exploratoin. original: 5x10^4 replay_size = 32 # Replay (batch) size hist_size = 2 # original: 4 data_index = 0 data_flag = False loss_log = '../playground/Assets/log/' def __init__(self, use_gpu, enable_controller, cnn_input_dim, feature_dim, agent_count, other_input_dim, model): self.use_gpu = use_gpu self.num_of_actions = len(enable_controller) self.enable_controller = enable_controller self.cnn_input_dim = cnn_input_dim self.feature_dim = feature_dim self.agent_count = agent_count self.other_input_dim = other_input_dim self.data_size = self.timestep_per_episode self.loss_log_file = self.loss_log + "loss.log" self.loss_per_episode = 0 self.time_of_episode = 0 print("Initializing Q-Network...") if model == 'None': self.model = Chain( conv1=L.Convolution2D(3 * self.hist_size, 32, 4, stride=2), bn1=L.BatchNormalization(32), conv2=L.Convolution2D(32, 32, 4, stride=2), bn2=L.BatchNormalization(32), conv3=L.Convolution2D(32, 32, 4, stride=2), bn3=L.BatchNormalization(32), # conv4=L.Convolution2D(64, 64, 4, stride=2), # bn4=L.BatchNormalization(64), l1=L.Linear( self.feature_dim + self.other_input_dim * self.hist_size, 128), l2=L.Linear(128, 128), l3=L.Linear(128, 96), l4=L.Linear(96, 64), q_value=L.Linear(64, self.num_of_actions)) else: with open(model, 'rb') as i: self.model = pickle.load(i) self.data_size = 0 if self.use_gpu >= 0: self.model.to_gpu() self.optimizer = optimizers.RMSpropGraves() self.optimizer.setup(self.model) # History Data : D=[s, a, r, s_dash, end_episode_flag] self.d = [ np.zeros((self.agent_count, self.data_size, self.hist_size, 128, 128, 3), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, self.hist_size, self.other_input_dim), dtype=np.uint8), np.zeros((self.agent_count, self.data_size), dtype=np.uint8), np.zeros((self.agent_count, self.data_size, 1), dtype=np.float32), np.zeros((self.agent_count, self.data_size, 1), dtype=np.bool) ] def _reshape_for_cnn(self, state, batch_size, hist_size, x, y): state_ = np.zeros((batch_size, 3 * hist_size, 128, 128), dtype=np.float32) for i in range(batch_size): if self.hist_size == 1: state_[i] = state[i][0].transpose(2, 0, 1) elif self.hist_size == 2: state_[i] = np.c_[state[i][0], state[i][1]].transpose(2, 0, 1) elif self.hist_size == 4: state_[i] = np.c_[state[i][0], state[i][1], state[i][2], state[i][3]].transpose(2, 0, 1) return state_ def forward(self, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end): num_of_batch = state_cnn.shape[0] s_cnn = Variable(state_cnn) s_oth = Variable(state_other) s_cnn_dash = Variable(state_cnn_dash) s_oth_dash = Variable(state_other_dash) q = self.q_func(s_cnn, s_oth) # Get Q-value max_q_dash_ = self.q_func(s_cnn_dash, s_oth_dash) if self.use_gpu >= 0: tmp = list(map(np.max, max_q_dash_.data.get())) else: tmp = list(map(np.max, max_q_dash_.data)) max_q_dash = np.asanyarray(tmp, dtype=np.float32) if self.use_gpu >= 0: target = np.array(q.data.get(), dtype=np.float32) else: target = np.array(q.data, dtype=np.float32) for i in range(num_of_batch): tmp_ = reward[i] + (1 - episode_end[i]) * self.gamma * max_q_dash[i] action_index = self.action_to_index(action[i]) target[i, action_index] = tmp_ if self.use_gpu >= 0: loss = F.mean_squared_error(Variable(cuda.to_gpu(target)), q) else: loss = F.mean_squared_error(Variable(target), q) return loss, q def stock_experience(self, time, state_cnn, state_other, action, reward, state_cnn_dash, state_other_dash, episode_end_flag): for i in range(self.agent_count): self.d[0][i][self.data_index] = state_cnn[i].copy() self.d[1][i][self.data_index] = state_other[i].copy() self.d[2][i][self.data_index] = action[i].copy() self.d[3][i][self.data_index] = reward[i].copy() self.d[4][i][self.data_index] = episode_end_flag self.data_index += 1 if self.data_index >= self.data_size: self.data_index -= self.data_size self.data_flag = True def experience_replay(self, time): if self.initial_exploration < time: # Pick up replay_size number of samples from the Data replayRobotIndex = np.random.randint(0, self.agent_count, self.replay_size) if not self.data_flag: # during the first sweep of the History Data replay_index = np.random.randint(0, self.data_index, self.replay_size) else: replay_index = np.random.randint(0, self.data_size, self.replay_size) s_cnn_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) a_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.uint8) r_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.float32) s_cnn_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, 128, 128, 3), dtype=np.float32) s_oth_dash_replay = np.ndarray(shape=(self.replay_size, self.hist_size, self.other_input_dim), dtype=np.float32) episode_end_replay = np.ndarray(shape=(self.replay_size, 1), dtype=np.bool) for i in range(self.replay_size): s_cnn_replay[i] = np.asarray( (self.d[0][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) s_oth_replay[i] = np.asarray( (self.d[1][replayRobotIndex[i]][replay_index[i]]), dtype=np.float32) a_replay[i] = self.d[2][replayRobotIndex[i]][replay_index[i]] r_replay[i] = self.d[3][replayRobotIndex[i]][replay_index[i]] if (replay_index[i] + 1 >= self.data_size): s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1 - self.data_size]), dtype=np.float32) else: s_cnn_dash_replay[i] = np.array( (self.d[0][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) s_oth_dash_replay[i] = np.array( (self.d[1][replayRobotIndex[i]][replay_index[i] + 1]), dtype=np.float32) episode_end_replay[i] = self.d[4][replayRobotIndex[i]][ replay_index[i]] s_cnn_replay = self._reshape_for_cnn(s_cnn_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_dash_replay = self._reshape_for_cnn(s_cnn_dash_replay, self.replay_size, self.hist_size, 128, 128) s_cnn_replay /= 255.0 s_oth_replay /= 255.0 s_cnn_dash_replay /= 255.0 s_oth_dash_replay /= 255.0 if self.use_gpu >= 0: s_cnn_replay = cuda.to_gpu(s_cnn_replay) s_oth_replay = cuda.to_gpu(s_oth_replay) s_cnn_dash_replay = cuda.to_gpu(s_cnn_dash_replay) s_oth_dash_replay = cuda.to_gpu(s_oth_dash_replay) # Gradient-based update loss, _ = self.forward(s_cnn_replay, s_oth_replay, a_replay, r_replay, s_cnn_dash_replay, s_oth_dash_replay, episode_end_replay) send_loss = loss.data with open(self.loss_log_file, 'a') as the_file: the_file.write(str(time) + "," + str(send_loss) + "\n") self.loss_per_episode += loss.data self.time_of_episode += 1 self.model.zerograds() loss.backward() self.optimizer.update() def q_func(self, state_cnn, state_other): if self.use_gpu >= 0: num_of_batch = state_cnn.data.get().shape[0] else: num_of_batch = state_cnn.data.shape[0] h1 = F.tanh(self.model.bn1(self.model.conv1(state_cnn))) h2 = F.tanh(self.model.bn2(self.model.conv2(h1))) h3 = F.tanh(self.model.bn3(self.model.conv3(h2))) # h4 = F.tanh(self.model.bn4(self.model.conv4(h3))) # h5 = F.tanh(self.model.bn5(self.model.conv5(h4))) h4_ = F.concat( (F.reshape(h3, (num_of_batch, self.feature_dim)), F.reshape(state_other, (num_of_batch, self.other_input_dim * self.hist_size))), axis=1) h6 = F.relu(self.model.l1(h4_)) h7 = F.relu(self.model.l2(h6)) h8 = F.relu(self.model.l3(h7)) h9 = F.relu(self.model.l4(h8)) q = self.model.q_value(h9) return q def e_greedy(self, state_cnn, state_other, epsilon, reward): s_cnn = Variable(state_cnn) s_oth = Variable(state_other) q = self.q_func(s_cnn, s_oth) q = q.data if self.use_gpu >= 0: q_ = q.get() else: q_ = q index_action = np.zeros((self.agent_count), dtype=np.uint8) print(("agent"), end=' ') for i in range(self.agent_count): if np.random.rand() < epsilon: index_action[i] = np.random.randint(0, self.num_of_actions) print(("[%02d] Random(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') else: index_action[i] = np.argmax(q_[i]) print(("[%02d]!Greedy(%2d)reward(%06.2f)" % (i, index_action[i], reward[i])), end=' ') if i % 5 == 4: print(("\n "), end=' ') del q_ return self.index_to_action(index_action), q def index_to_action(self, index_of_action): index = np.zeros((self.agent_count), dtype=np.uint8) for i in range(self.agent_count): index[i] = self.enable_controller[index_of_action[i]] return index def action_to_index(self, action): return self.enable_controller.index(action)
class QNet: # Hyper-Parameters gamma = 0.99 # Discount factor initial_exploration = 10**3 # Initial exploratoin. original: 5x10^4 replay_size = 32 # Replay (batch) size target_model_update_freq = 10**4 # Target update frequancy. original: 10^4 data_size = 10**5 # Data size of history. original: 10^6 hist_size = 1 # original: 4 def __init__(self, use_gpu, enable_controller, dim, epsilon, epsilon_delta, min_eps): self.use_gpu = use_gpu self.num_of_actions = len(enable_controller) self.enable_controller = enable_controller self.dim = dim self.epsilon = epsilon self.epsilon_delta = epsilon_delta self.min_eps = min_eps self.time = 0 app_logger.info("Initializing Q-Network...") hidden_dim = 256 self.model = Chain( l4=L.Linear(self.dim * self.hist_size, hidden_dim, initialW=initializers.Normal( 0.5 / math.sqrt(self.dim * self.hist_size))), q_value=L.Linear(hidden_dim, self.num_of_actions, initialW=np.zeros( (self.num_of_actions, hidden_dim), dtype=np.float32))) if self.use_gpu >= 0: self.model.to_gpu() self.model_target = copy.deepcopy(self.model) self.optimizer = optimizers.RMSpropGraves(lr=0.00025, alpha=0.95, momentum=0.95, eps=0.0001) self.optimizer.setup(self.model) # History Data : D=[s, a, r, s_dash, end_episode_flag] self.d = [ np.zeros((self.data_size, self.hist_size, self.dim), dtype=np.uint8), np.zeros(self.data_size, dtype=np.uint8), np.zeros((self.data_size, 1), dtype=np.int8), np.zeros((self.data_size, self.hist_size, self.dim), dtype=np.uint8), np.zeros((self.data_size, 1), dtype=np.bool) ] def forward(self, state, action, reward, state_dash, episode_end): num_of_batch = state.shape[0] s = Variable(state) s_dash = Variable(state_dash) q = self.q_func(s) # Get Q-value # Generate Target Signals tmp = self.q_func_target(s_dash) # Q(s',*) if self.use_gpu >= 0: tmp = list(map(np.max, tmp.data.get())) # max_a Q(s',a) else: tmp = list(map(np.max, tmp.data)) # max_a Q(s',a) max_q_dash = np.asanyarray(tmp, dtype=np.float32) if self.use_gpu >= 0: target = np.asanyarray(q.data.get(), dtype=np.float32) else: # make new array target = np.array(q.data, dtype=np.float32) for i in xrange(num_of_batch): if not episode_end[i][0]: tmp_ = reward[i] + self.gamma * max_q_dash[i] else: tmp_ = reward[i] action_index = self.action_to_index(action[i]) target[i, action_index] = tmp_ # TD-error clipping if self.use_gpu >= 0: target = cuda.to_gpu(target) td = Variable(target) - q # TD error td_tmp = td.data + 1000.0 * (abs(td.data) <= 1) # Avoid zero division td_clip = td * (abs(td.data) <= 1) + td / abs(td_tmp) * (abs(td.data) > 1) zero_val = np.zeros((self.replay_size, self.num_of_actions), dtype=np.float32) if self.use_gpu >= 0: zero_val = cuda.to_gpu(zero_val) zero_val = Variable(zero_val) loss = F.mean_squared_error(td_clip, zero_val) return loss, q def q_func(self, state): h4 = F.relu(self.model.l4(state / 255.0)) q = self.model.q_value(h4) return q def q_func_target(self, state): h4 = F.relu(self.model_target.l4(state / 255.0)) q = self.model_target.q_value(h4) return q def e_greedy(self, state, epsilon): s = Variable(state) q = self.q_func(s) q = q.data if np.random.rand() < epsilon: index_action = np.random.randint(0, self.num_of_actions) app_logger.info(" Random") else: if self.use_gpu >= 0: index_action = np.argmax(q.get()) else: index_action = np.argmax(q) app_logger.info("#Greedy") return self.index_to_action(index_action), q def target_model_update(self): self.model_target = copy.deepcopy(self.model) def index_to_action(self, index_of_action): return self.enable_controller[index_of_action] def action_to_index(self, action): return self.enable_controller.index(action) def start(self, feature): self.state = np.zeros((self.hist_size, self.dim), dtype=np.uint8) self.state[0] = feature state_ = np.asanyarray(self.state.reshape(1, self.hist_size, self.dim), dtype=np.float32) if self.use_gpu >= 0: state_ = cuda.to_gpu(state_) # Generate an Action e-greedy action, q_now = self.e_greedy(state_, self.epsilon) return_action = action return return_action def update_model(self, replayed_experience): if replayed_experience[0]: self.model.zerograds() loss, _ = self.forward(replayed_experience[1], replayed_experience[2], replayed_experience[3], replayed_experience[4], replayed_experience[5]) loss.backward() self.optimizer.update() # Target model update if replayed_experience[0] and np.mod( self.time, self.target_model_update_freq) == 0: app_logger.info("Model Updated") self.target_model_update() self.time += 1 app_logger.info("step: {}".format(self.time)) def step(self, features): if self.hist_size == 4: self.state = np.asanyarray( [self.state[1], self.state[2], self.state[3], features], dtype=np.uint8) elif self.hist_size == 2: self.state = np.asanyarray([self.state[1], features], dtype=np.uint8) elif self.hist_size == 1: self.state = np.asanyarray([features], dtype=np.uint8) else: app_logger.error("self.DQN.hist_size err") state_ = np.asanyarray(self.state.reshape(1, self.hist_size, self.dim), dtype=np.float32) if self.use_gpu >= 0: state_ = cuda.to_gpu(state_) # Exploration decays along the time sequence if self.initial_exploration < self.time: self.epsilon -= self.epsilon_delta if self.epsilon < self.min_eps: self.epsilon = self.min_eps eps = self.epsilon else: # Initial Exploation Phase app_logger.info("Initial Exploration : {}/{} steps".format( self.time, self.initial_exploration)) eps = 1.0 # Generate an Action by e-greedy action selection action, q_now = self.e_greedy(state_, eps) if self.use_gpu >= 0: q_max = np.max(q_now.get()) else: q_max = np.max(q_now) return action, eps, q_max
class nfp(object): """NFP manager This class has the generator function of NFP and updator of NN for learning the generator of NFP. Args: d: Dimension of NFP. f: Dimension of the feature for generating NFP. R: Radius for generating NFP. """ def __init__(self, d, f, R): self.d = d self.f = f self.R = R g = ChainList(*[L.Linear(1, f) for i in six.moves.range(AtomIdMax)]) H = ChainList(*[ ChainList(*[L.Linear(f, f) for i in six.moves.range(R)]) for j in six.moves.range(5) ]) W = ChainList(*[L.Linear(f, d) for i in six.moves.range(R)]) self.model = Chain(H=H, W=W, g=g) self.optimizer = optimizers.Adam() self.optimizer.setup(self.model) def get_nfp(self, sid, train=True): """Generates NFP. Args: sid (int): Substance ID. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: fp: NFP. """ d, f, R = self.d, self.f, self.R mol = data.load_sdf(sid) atoms = mol.GetAtoms() n = len(atoms) fp = Variable(np.zeros([1, d], dtype='float32'), volatile=not train) r = [[ Variable(np.zeros([1, f], dtype='float32'), volatile=not train) for i in six.moves.range(n) ] for j in six.moves.range(R + 1)] for atom in atoms: a = atom.GetIdx() anum = atom.GetAtomicNum() r[0][a] += self.model.g[anum](Variable(np.array([[1]], dtype='float32'), volatile=not train)) for l in six.moves.range(R): v = [ Variable(np.zeros([1, f], dtype='float32'), volatile=not train) for i in six.moves.range(n) ] for atom in atoms: a = atom.GetIdx() v[a] += r[l][a] for n_atom in atom.GetNeighbors(): na = n_atom.GetIdx() v[a] += r[l][na] for atom in atoms: a = atom.GetIdx() deg = atom.GetDegree() deg = min(5, max(1, deg)) r[l + 1][a] = F.tanh(self.model.H[deg - 1][l](v[a])) i = F.softmax(self.model.W[l](r[l + 1][a])) fp += i return fp def update(self, sids, y, net, train=True): """Updates NFP NN. Args: sids (int[]): Substance ID. y (np.array(int32[])[2]): Activity data. y[0] is for the training dataset and y[1] is for the test dataset. net (nn.NN): Classifier of QSAR. train (boolean): Training flag. If you want to train the NFP NN, set it True, otherwise False. Returns: result (float): Overall accuracy on the test dataset. """ def get_nfps(sids, train=True): print('generate fingerprints...') fps = {} for i, sid in enumerate(sids[0] + sids[1]): fps[sid] = self.get_nfp(sid, train) print('done.') return fps self.model.zerograds() fps = get_nfps(sids, train) x_train = [fps[sid] for sid in sids[0]] x_test = [fps[sid] for sid in sids[1]] for x in x_train: x.volatile = 'off' for x in x_test: x.volatile = 'off' result = net.train(x_train, y[0], x_test, y[1], train) self.optimizer.update() return result
def model(x): y = F.sigmoid(NNset.l1(x)) return y x = Variable(xtrain) t = Variable(ytrain) optimizer = optimizers.SGD() optimizer.setup(NNset) Tall = 1000 train_loss = [] for i in range(Tall): NNset.zerograds() y = model(x) loss = F.mean_squared_error(y,t) loss.backward() optimizer.update() train_loss.append(loss.data) plt.figure(figsize=(8,6)) plt.plot(range(Tall),train_loss) plt.title('optimization vol2') plt.xlabel('step') plt.ylabel('loss function') plt.xlim([0,Tall]) plt.ylim([0,0.5]) plt.show() <<<<<<< HEAD:iris_SGD.py