def _init(self, ob_space, ac_space, kind): assert isinstance(ob_space, gym.spaces.Box) self.pdtype = pdtype = make_pdtype(ac_space) sequence_length = None ob = U.get_placeholder(name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape)) x = ob / 255.0 if kind == 'small': # from A3C paper x = tf.nn.relu(U.conv2d(x, 16, "l1", [8, 8], [4, 4], pad="VALID")) x = tf.nn.relu(U.conv2d(x, 32, "l2", [4, 4], [2, 2], pad="VALID")) x = U.flattenallbut0(x) x = tf.nn.relu(U.dense(x, 256, 'lin', U.normc_initializer(1.0))) elif kind == 'large': # Nature DQN x = tf.nn.relu(U.conv2d(x, 32, "l1", [8, 8], [4, 4], pad="VALID")) x = tf.nn.relu(U.conv2d(x, 64, "l2", [4, 4], [2, 2], pad="VALID")) x = tf.nn.relu(U.conv2d(x, 64, "l3", [3, 3], [1, 1], pad="VALID")) x = U.flattenallbut0(x) x = tf.nn.relu(U.dense(x, 512, 'lin', U.normc_initializer(1.0))) else: raise NotImplementedError logits = U.dense(x, pdtype.param_shape()[0], "logits", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat(logits) self.vpred = U.dense(x, 1, "value", U.normc_initializer(1.0))[:,0] self.state_in = [] self.state_out = [] stochastic = tf.placeholder(dtype=tf.bool, shape=()) ac = self.pd.sample() # XXX self._act = U.function([stochastic, ob], [ac, self.vpred])
def _init(self, ob_space, ac_space, hid_size, num_hid_layers, gaussian_fixed_var=True): assert isinstance(ob_space, gym.spaces.Box) self.pdtype = pdtype = make_pdtype(ac_space) sequence_length = None ob = U.get_placeholder(name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape)) with tf.variable_scope("obfilter"): self.ob_rms = RunningMeanStd(shape=ob_space.shape) obz = tf.clip_by_value((ob - self.ob_rms.mean) / self.ob_rms.std, -5.0, 5.0) last_out = obz for i in range(num_hid_layers): last_out = tf.nn.tanh(U.dense(last_out, hid_size, "vffc%i"%(i+1), weight_init=U.normc_initializer(1.0))) self.vpred = U.dense(last_out, 1, "vffinal", weight_init=U.normc_initializer(1.0))[:,0] last_out = obz for i in range(num_hid_layers): last_out = tf.nn.tanh(U.dense(last_out, hid_size, "polfc%i"%(i+1), weight_init=U.normc_initializer(1.0))) if gaussian_fixed_var and isinstance(ac_space, gym.spaces.Box): mean = U.dense(last_out, pdtype.param_shape()[0]//2, "polfinal", U.normc_initializer(0.01)) logstd = tf.get_variable(name="logstd", shape=[1, pdtype.param_shape()[0]//2], initializer=tf.zeros_initializer()) pdparam = U.concatenate([mean, mean * 0.0 + logstd], axis=1) else: pdparam = U.dense(last_out, pdtype.param_shape()[0], "polfinal", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat(pdparam) self.state_in = [] self.state_out = [] # change for BC #stochastic = tf.placeholder(dtype=tf.bool, shape=()) stochastic = U.get_placeholder(name="stochastic", dtype=tf.bool, shape=()) ac = U.switch(stochastic, self.pd.sample(), self.pd.mode()) self.ac = ac self._act = U.function([stochastic, ob], [ac, self.vpred])
def _init(self, ob_space, ac_space): assert isinstance(ob_space, gym.spaces.Box) self.pdtype = pdtype = make_pdtype(ac_space) sequence_length = None ob = U.get_placeholder(name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape)) obscaled = ob / 255.0 with tf.variable_scope("pol"): x = obscaled x = tf.nn.relu(U.conv2d(x, 8, "l1", [8, 8], [4, 4], pad="VALID")) x = tf.nn.relu(U.conv2d(x, 16, "l2", [4, 4], [2, 2], pad="VALID")) x = U.flattenallbut0(x) x = tf.nn.relu(U.dense(x, 128, 'lin', U.normc_initializer(1.0))) logits = U.dense(x, pdtype.param_shape()[0], "logits", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat(logits) with tf.variable_scope("vf"): x = obscaled x = tf.nn.relu(U.conv2d(x, 8, "l1", [8, 8], [4, 4], pad="VALID")) x = tf.nn.relu(U.conv2d(x, 16, "l2", [4, 4], [2, 2], pad="VALID")) x = U.flattenallbut0(x) x = tf.nn.relu(U.dense(x, 128, 'lin', U.normc_initializer(1.0))) self.vpred = U.dense(x, 1, "value", U.normc_initializer(1.0)) self.vpredz = self.vpred self.state_in = [] self.state_out = [] stochastic = tf.placeholder(dtype=tf.bool, shape=()) ac = self.pd.sample() # XXX self._act = U.function([stochastic, ob], [ac, self.vpred])
def _init(self, ob_space, ac_space, hid_size, num_hid_layers, gaussian_fixed_var=True): assert isinstance( ob_space, gym.spaces.Box) #ru guo hou mian tiao jian wei jia ze tui chu #print ("mlp_policy/20lines") zhi xing liang ci #print ("ac_space.shape[0]", ac_space.shape[0]) shu chu jie guo shi 3 self.pdtype = pdtype = make_pdtype( ac_space ) #return DiagGaussianPdType(ac_space.shape[0]) zhe li mian zui hou you pdclass() sequence_length = None ob = U.get_placeholder( name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape) ) #return tf.placeholder(dtype=dtype, shape=shape, name=name) #print ("obspace.shape:::", list(ob_space.shape)) shu chu shi [11] with tf.variable_scope("obfilter"): #print("gail-tf/gailtf/baselines/ppo1/mlp_policy.py/28lines:") self.ob_rms = RunningMeanStd( shape=ob_space.shape) #zhe ge han shu kan bu dong obz = tf.clip_by_value((ob - self.ob_rms.mean) / self.ob_rms.std, -5.0, 5.0) #ob zhe ge shi hou hai shi placeholder last_out = obz for i in range(num_hid_layers): last_out = tf.nn.tanh( U.dense(last_out, hid_size, "vffc%i" % (i + 1), weight_init=U.normc_initializer( 1.0))) #da jian le quan lian jie ceng self.vpred = U.dense( last_out, 1, "vffinal", weight_init=U.normc_initializer(1.0) )[:, 0] #wen ti shi zhe li zui hou mei you shu chu dong zuo de kongjian last_out = obz for i in range(num_hid_layers): last_out = tf.nn.tanh( U.dense(last_out, hid_size, "polfc%i" % (i + 1), weight_init=U.normc_initializer(1.0))) if gaussian_fixed_var and isinstance(ac_space, gym.spaces.Box): print("gaussian_fixed_var is used") mean = U.dense(last_out, pdtype.param_shape()[0] // 2, "polfinal", U.normc_initializer(0.01)) logstd = tf.get_variable(name="logstd", shape=[1, pdtype.param_shape()[0] // 2], initializer=tf.zeros_initializer()) pdparam = U.concatenate([mean, mean * 0.0 + logstd], axis=1) else: #print ("gaussian_fixed_var is not used") mei you bei yong dao pdparam = U.dense(last_out, pdtype.param_shape()[0], "polfinal", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat( pdparam ) # mo rren shang mian de pdtype yi ding shi DiagGaussianPd return DiagGaussianPd #pd li mian you kl, entropy, sample deng fang fa self.state_in = [] self.state_out = [] # change for BC #stochastic = tf.placeholder(dtype=tf.bool, shape=()) stochastic = U.get_placeholder(name="stochastic", dtype=tf.bool, shape=()) ac = U.switch(stochastic, self.pd.sample(), self.pd.mode()) self.ac = ac self._act = U.function([stochastic, ob], [ac, self.vpred])
def _init(self, ob_space, ac_space, hid_size, num_hid_layers, gaussian_fixed_var=True): assert isinstance(ob_space, gym.spaces.Box) self.pdtype = pdtype = make_pdtype(ac_space) sequence_length = None ob = U.get_placeholder(name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape)) last_action = U.get_placeholder(shape=(None, 524), dtype=tf.float32, name="last_action_one_hot") self.msize = 64 # change to 64 later self.ssize = 64 self.isize = 11 self.available_action_size = 524 available_action = ob[:, (5*self.msize*self.msize+10*self.ssize*self.ssize+self.isize):(5*self.msize*self.msize+10*self.ssize*self.ssize+self.isize+self.available_action_size)] # ob = ob[:,:-(self.available_action_size)] with tf.variable_scope("obfilter"): self.ob_rms = RunningMeanStd(shape=ob_space.shape) # obz = tf.clip_by_value((ob - self.ob_rms.mean) / self.ob_rms.std, -20.0, 20.0) obz = (ob - self.ob_rms.mean) / self.ob_rms.std minimap = obz[:, 0:5*self.msize*self.msize] # minimap /= 2 screen = obz[:, 5*self.msize*self.msize: 5*self.msize*self.msize+ 10*self.ssize*self.ssize] # screen /= 2 info = obz[:, (5*self.msize*self.msize+10*self.ssize*self.ssize):(5*self.msize*self.msize+10*self.ssize*self.ssize+self.isize)] # info /= 2 # get value prediction, crtic mconv1 = tf.layers.conv2d( inputs=tf.reshape(minimap, [-1,self.msize,self.msize,5]), filters=32, kernel_size=[5, 5], padding="same", kernel_initializer=U.normc_initializer(0.01), activation=tf.nn.leaky_relu) mpool1 = tf.layers.max_pooling2d(inputs=mconv1, pool_size=[2, 2], strides=2) mconv2 = tf.layers.conv2d( inputs=mpool1, filters=64, kernel_size=[5, 5], padding="same", kernel_initializer=U.normc_initializer(0.01), activation=tf.nn.leaky_relu, name="vffcmconv2") mpool2 = tf.layers.max_pooling2d(inputs=mconv2, pool_size=[2, 2], strides=2) mpool2_flat = tf.reshape(mpool2, [-1, 16 * 16 * 64]) sconv1 = tf.layers.conv2d( inputs=tf.reshape(screen, [-1,self.ssize, self.ssize,10]), filters=48, kernel_size=[5, 5], padding="same", kernel_initializer=U.normc_initializer(0.01), activation=tf.nn.leaky_relu) spool1 = tf.layers.max_pooling2d(inputs=sconv1, pool_size=[2, 2], strides=2) sconv2 = tf.layers.conv2d( inputs=spool1, filters=80, kernel_size=[5, 5], padding="same", kernel_initializer=U.normc_initializer(0.01), activation=tf.nn.leaky_relu) spool2 = tf.layers.max_pooling2d(inputs=sconv2, pool_size=[2, 2], strides=2) spool2_flat = tf.reshape(spool2, [-1, 16 * 16 * 80]) info_fc = tf.layers.dense(inputs=layers.flatten(info), units=8, activation=tf.tanh) aa_fc = tf.layers.dense(inputs=layers.flatten(available_action), units=32, activation=tf.tanh) HIDDEN_SIZE = 128 l1_action = tf.layers.dense(layers.flatten(last_action), 256, tf.nn.relu) input_to_rnn = tf.reshape(l1_action, [-1, 16, 16]) action_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=HIDDEN_SIZE, forget_bias=1.0, state_is_tuple=True) inputs_rnn = tf.unstack(input_to_rnn, num=16, axis=1) rnn_outputs,rnn_state= tf.contrib.rnn.static_rnn(action_lstm_cell, inputs_rnn, dtype=tf.float32) l2_action = tf.layers.dense(rnn_state[-1], 128, tf.nn.tanh) # hidden layer last_acs_ph_lstm = tf.layers.dense(l2_action, 32, tf.nn.tanh) last_out = tf.concat([mpool2_flat, spool2_flat, info_fc, aa_fc, last_acs_ph_lstm], axis=1) vf_last_out = tf.nn.tanh(U.dense(last_out, 1024, 'vf_last_out', weight_init=U.normc_initializer(1.0))) # vf_last_out_2 = tf.nn.tanh(U.dense(vf_last_out, 64, 'vf_last_out_2', # weight_init=U.normc_initializer(1.0))) self.vpred = U.dense(vf_last_out, 1, "vffinal", weight_init=U.normc_initializer(1.0))[:,0] if gaussian_fixed_var and isinstance(ac_space, gym.spaces.Box): mean = U.dense(last_out, pdtype.param_shape()[0]//2, "polfinal", U.normc_initializer(0.01)) logstd = tf.get_variable(name="logstd", shape=[1, pdtype.param_shape()[0]//2], initializer=tf.zeros_initializer()) pdparam = U.concatenate([mean, mean * 0.0 + logstd], axis=1) else: pol_last_out = U.dense(last_out, (pdtype.param_shape()[0])*5, "polfinaldense", U.normc_initializer(0.01)) pdparam = U.dense(pol_last_out, pdtype.param_shape()[0], "polfinal", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat(pdparam) self.state_in = [] self.state_out = [] # change for BC #stochastic = tf.placeholder(dtype=tf.bool, shape=()) stochastic = U.get_placeholder(name="stochastic", dtype=tf.bool, shape=()) ac = U.switch(stochastic, self.pd.sample(available_action), self.pd.mode(available_action)) self.ac = ac self._act = U.function([stochastic, ob, last_action], [ac, self.vpred])
def _init(self, ob_space, ac_space, hid_size, num_hid_layers, gaussian_fixed_var=True): assert isinstance(ob_space, gym.spaces.Box) self.pdtype = pdtype = make_pdtype(ac_space) sequence_length = None ob = U.get_placeholder(name="ob", dtype=tf.float32, shape=[sequence_length] + list(ob_space.shape)) with tf.variable_scope("obfilter"): self.ob_rms = RunningMeanStd(shape=ob_space.shape) obz = tf.clip_by_value((ob - self.ob_rms.mean) / self.ob_rms.std, -20.0, 20.0) last_out = obz for i in range(num_hid_layers): last_out = tf.nn.tanh( U.dense(last_out, hid_size, "vffc%i" % (i + 1), weight_init=U.normc_initializer(1.0))) self.vpred = U.dense(last_out, 1, "vffinal", weight_init=U.normc_initializer(1.0))[:, 0] # last_out = obz # for i in range(num_hid_layers): # last_out = tf.nn.tanh(U.dense(last_out, hid_size, "polfc%i"%(i+1), weight_init=U.normc_initializer(1.0))) ### add conv net instead of using dense self.msize = 64 # change to 64 later self.ssize = 64 self.isize = 11 self.available_action_size = 524 minimap = obz[:, 0:5 * self.msize * self.msize] screen = obz[:, 5 * self.msize * self.msize:5 * self.msize * self.msize + 10 * self.ssize * self.ssize] info = obz[:, (5 * self.msize * self.msize + 10 * self.ssize * self.ssize):( 5 * self.msize * self.msize + 10 * self.ssize * self.ssize + self.isize)] available_action = obz[:, (5 * self.msize * self.msize + 10 * self.ssize * self.ssize + self.isize):(5 * self.msize * self.msize + 10 * self.ssize * self.ssize + self.isize + self.available_action_size)] conv1_minimap = tf.layers.conv2d(inputs=tf.reshape( minimap, [-1, self.msize, self.msize, 5]), filters=10, kernel_size=5, strides=1, padding='same', activation=tf.nn.leaky_relu, name="polmconv1") # -> (64, 64, 10) pool1_minimap = tf.layers.max_pooling2d( conv1_minimap, pool_size=4, strides=4, name="polmpool1") # -> (16, 16, 10) conv2_minimap = tf.layers.conv2d(pool1_minimap, 10, 5, 1, 'same', activation=tf.nn.relu, name="polmconv2") # -> (16, 16, 10) pool2_minimap = tf.layers.max_pooling2d( conv2_minimap, 2, 2, name="polmpool2") # -> (8, 8, 10) flat_minimap = tf.reshape(pool2_minimap, [-1, 8 * 8 * 10]) # -> (8*8*10, ) # dense_minimap = tf.layers.dense(inputs=flat_minimap, units=1024, activation=tf.nn.relu) # # dropout_mininmap = tf.layers.dropout( # # inputs=dense_minimap, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) # minimap_output = tf.layers.dense(dense_minimap, 64) conv1_screen = tf.layers.conv2d( inputs=tf.reshape(screen, [-1, self.ssize, self.ssize, 10]), # (64,64,10) filters=20, kernel_size=5, strides=1, padding='same', activation=tf.nn.leaky_relu, name="polsconv1") # -> (64, 64, 20) pool1_screen = tf.layers.max_pooling2d( conv1_screen, pool_size=4, strides=4, name="polspool1") # -> (16, 16, 20) conv2_screen = tf.layers.conv2d(pool1_screen, 20, 5, 1, 'same', activation=tf.nn.relu, name="polsconv2") # -> (16, 16, 20) pool2_screen = tf.layers.max_pooling2d( conv2_screen, 2, 2, name="polspool2") # -> (8, 8, 20) flat_screen = tf.reshape(pool2_screen, [-1, 8 * 8 * 20]) # -> (8*8*20, ) # dense_screen = tf.layers.dense(inputs=flat_screen, units=1024, activation=tf.nn.relu) # # dropout_screen = tf.layers.dropout( # # inputs=dense_screen, rate=0.4, training=mode == tf.estimator.ModeKeys.TRAIN) # screen_output = tf.layers.dense(dense_screen, 64, tf.nn.relu) info_fc = tf.layers.dense(inputs=layers.flatten(info), units=4, activation=tf.tanh, name="poldense1") aa_fc = tf.layers.dense(inputs=layers.flatten(available_action), units=16, activation=tf.tanh, name="poldense2") last_out = tf.concat([flat_minimap, flat_screen, info_fc, aa_fc], axis=1, name="polconcat") # last_out = tf.layers.dense(inputs=last_out,units=600,name="poldense3") # last_out = tf.nn.tanh(U.dense(last_out, hid_size, "polfc1", weight_init=U.normc_initializer(1.0))) if gaussian_fixed_var and isinstance(ac_space, gym.spaces.Box): mean = U.dense(last_out, pdtype.param_shape()[0] // 2, "polfinal", U.normc_initializer(0.01)) logstd = tf.get_variable(name="logstd", shape=[1, pdtype.param_shape()[0] // 2], initializer=tf.zeros_initializer()) pdparam = U.concatenate([mean, mean * 0.0 + logstd], axis=1) else: pdparam = U.dense(last_out, pdtype.param_shape()[0], "polfinal", U.normc_initializer(0.01)) self.pd = pdtype.pdfromflat(pdparam) self.state_in = [] self.state_out = [] # change for BC #stochastic = tf.placeholder(dtype=tf.bool, shape=()) stochastic = U.get_placeholder(name="stochastic", dtype=tf.bool, shape=()) ac = U.switch(stochastic, self.pd.sample(), self.pd.mode()) self.ac = ac self._act = U.function([stochastic, ob], [ac, self.vpred])