def q_train_ddpg(observPlaceHolderList, actionSpaceList, q_index, q_func, optimizer, grad_norm_clipping=None, scope="trainer", reuse=None, num_units=64): with tf.variable_scope(scope, reuse=reuse): actionPlaceHolderTypeList = [make_pdtype(actionSpace) for actionSpace in actionSpaceList] # set up placeholders act_ph_n = [actionPlaceHolderTypeList[i].sample_placeholder([None], name="action"+str(i)) for i in range(len(actionSpaceList))] target_ph = tf.placeholder(tf.float32, [None], name="target") # q_input = tf.concat(observPlaceHolderList + act_ph_n, 1) q_input = tf.concat([observPlaceHolderList[q_index], act_ph_n[q_index]], 1) # specific for ddpg q = q_func(q_input, 1, scope="q_func", num_units=num_units)[:,0] q_func_vars = U.scope_vars(U.absolute_scope_name("q_func")) q_loss = tf.reduce_mean(tf.square(q - target_ph)) loss = q_loss #+ 1e-3 * q_reg optimize_expr = U.minimize_and_clip(optimizer, loss, q_func_vars, grad_norm_clipping) # Create callable functions train = U.function(inputs=observPlaceHolderList + act_ph_n + [target_ph], outputs=loss, updates=[optimize_expr]) q_values = U.function(observPlaceHolderList + act_ph_n, q) # target network target_q = q_func(q_input, 1, scope="target_q_func", num_units=num_units)[:,0] target_q_func_vars = U.scope_vars(U.absolute_scope_name("target_q_func")) update_target_q = make_update_exp(q_func_vars, target_q_func_vars) target_q_values = U.function(observPlaceHolderList + act_ph_n, target_q) return train, update_target_q, {'q_values': q_values, 'target_q_values': target_q_values}
def p_train(make_obs_ph_n, act_space_n, p_index, p_func, q_func, optimizer, grad_norm_clipping=None, local_q_func=False, num_units=64, scope="trainer", reuse=None): with tf.variable_scope(scope, reuse=reuse): # create distribtuions act_pdtype_n = [make_pdtype(act_space) for act_space in act_space_n] # SoftCategoricalPdType Object # set up placeholders obs_ph_n = make_obs_ph_n act_ph_n = [act_pdtype_n[i].sample_placeholder([None], name="action"+str(i)) for i in range(len(act_space_n))] # 0th element: name = 'agent_0_1/action0:0', shape = (?, 5) p_input = obs_ph_n[p_index] # mlp_model(tensor(,12), 5, scope="p_func", num_units=64) p = p_func(p_input, int(act_pdtype_n[p_index].param_shape()[0]), scope="p_func", num_units=num_units) p_func_vars = U.scope_vars(U.absolute_scope_name("p_func")) # wrap parameters in distribution act_pd = act_pdtype_n[p_index].pdfromflat(p) # SoftCategoricalPd Object act_sample = act_pd.sample() p_reg = tf.reduce_mean(tf.square(act_pd.flatparam())) act_input_n = act_ph_n + [] act_input_n[p_index] = act_pd.sample() q_input = tf.concat(obs_ph_n + act_input_n, 1) # shape = 12+12+10+5*3 if local_q_func: # ddpg, uses only personal obs/ act q_input = tf.concat([obs_ph_n[p_index], act_input_n[p_index]], 1) # shape = 17 = 12+ 5 q = q_func(q_input, 1, scope="q_func", reuse=True, num_units=num_units)[:,0] pg_loss = -tf.reduce_mean(q) loss = pg_loss + p_reg * 1e-3 optimize_expr = U.minimize_and_clip(optimizer, loss, p_func_vars, grad_norm_clipping) # Create callable functions train = U.function(inputs=obs_ph_n + act_ph_n, outputs=loss, updates=[optimize_expr]) act = U.function(inputs=[obs_ph_n[p_index]], outputs=act_sample) p_values = U.function([obs_ph_n[p_index]], p) # target network target_p = p_func(p_input, int(act_pdtype_n[p_index].param_shape()[0]), scope="target_p_func", num_units=num_units) target_p_func_vars = U.scope_vars(U.absolute_scope_name("target_p_func")) update_target_p = make_update_exp(p_func_vars, target_p_func_vars) target_act_sample = act_pdtype_n[p_index].pdfromflat(target_p).sample() target_act = U.function(inputs=[obs_ph_n[p_index]], outputs=target_act_sample) return act, train, update_target_p, {'p_values': p_values, 'target_act': target_act}
def make_update_exp(vals, target_vals): polyak = 1.0 - 1e-2 expression = [] for var, var_target in zip(sorted(vals, key=lambda v: v.name), sorted(target_vals, key=lambda v: v.name)): expression.append(var_target.assign(polyak * var_target + (1.0-polyak) * var)) expression = tf.group(*expression) return U.function([], [], updates=[expression])
def p_train(observPlaceHolderList, actionSpaceList, agentIndex, getMLPModel, q_func, optimizer, grad_norm_clipping=None, ddpg=False, num_units=64, scope="trainer", reuse=None): with tf.variable_scope(scope, reuse=reuse): # create distribtuions actionPlaceHolderTypeList = [make_pdtype(actionSpace) for actionSpace in actionSpaceList] # set up placeholders act_ph_n = [actionPlaceHolderTypeList[i].sample_placeholder([None], name="action"+str(i)) for i in range(len(actionSpaceList))] p_input = observPlaceHolderList[agentIndex] p = getMLPModel(p_input, int(actionPlaceHolderTypeList[agentIndex].param_shape()[0]), scope="getMLPModel", num_units=num_units) p_func_vars = U.scope_vars(U.absolute_scope_name("getMLPModel")) # wrap parameters in distribution act_pd = actionPlaceHolderTypeList[agentIndex].pdfromflat(p) act_sample = act_pd.sample() p_reg = tf.reduce_mean(tf.square(act_pd.flatparam())) act_input_n = act_ph_n + [] act_input_n[agentIndex] = act_pd.sample() q_input = tf.concat(observPlaceHolderList + act_input_n, 1) if ddpg: q_input = tf.concat([observPlaceHolderList[agentIndex], act_input_n[agentIndex]], 1) q = q_func(q_input, 1, scope="q_func", reuse=True, num_units=num_units)[:,0] pg_loss = -tf.reduce_mean(q) loss = pg_loss + p_reg * 1e-3 optimize_expr = U.minimize_and_clip(optimizer, loss, p_func_vars, grad_norm_clipping) # Create callable functions train = U.function(inputs=observPlaceHolderList + act_ph_n, outputs=loss, updates=[optimize_expr]) act = U.function(inputs=[observPlaceHolderList[agentIndex]], outputs=act_sample) p_values = U.function([observPlaceHolderList[agentIndex]], p) # target network target_p = getMLPModel(p_input, int(actionPlaceHolderTypeList[agentIndex].param_shape()[0]), scope="target_p_func", num_units=num_units) target_p_func_vars = U.scope_vars(U.absolute_scope_name("target_p_func")) update_target_p = make_update_exp(p_func_vars, target_p_func_vars) target_act_sample = actionPlaceHolderTypeList[agentIndex].pdfromflat(target_p).sample() target_act = U.function(inputs=[observPlaceHolderList[agentIndex]], outputs=target_act_sample) return act, train, update_target_p, {'p_values': p_values, 'target_act': target_act}
def q_train(make_obs_ph_n, act_space_n, q_index, q_func, optimizer, grad_norm_clipping=None, local_q_func=False, scope="trainer", reuse=None, num_units=64): with tf.variable_scope(scope, reuse=reuse): # parser.add_argument("--num-units", type=int, default=64, help="number of units in the mlp") # local_q_func = False if maddpgAlgor, = true if ddpg # create distribtuions act_pdtype_n = [make_pdtype(act_space) for act_space in act_space_n] # set up placeholders obs_ph_n = make_obs_ph_n act_ph_n = [act_pdtype_n[i].sample_placeholder([None], name="action"+str(i)) for i in range(len(act_space_n))] target_ph = tf.placeholder(tf.float32, [None], name="target") q_input = tf.concat(obs_ph_n + act_ph_n, 1) if local_q_func: q_input = tf.concat([obs_ph_n[q_index], act_ph_n[q_index]], 1) q = q_func(q_input, 1, scope="q_func", num_units=num_units)[:,0] q_func_vars = U.scope_vars(U.absolute_scope_name("q_func")) q_loss = tf.reduce_mean(tf.square(q - target_ph)) # viscosity solution to Bellman differential equation in place of an initial condition q_reg = tf.reduce_mean(tf.square(q)) loss = q_loss #+ 1e-3 * q_reg optimize_expr = U.minimize_and_clip(optimizer, loss, q_func_vars, grad_norm_clipping) # Create callable functions train = U.function(inputs=obs_ph_n + act_ph_n + [target_ph], outputs=loss, updates=[optimize_expr]) q_values = U.function(obs_ph_n + act_ph_n, q) # target network target_q = q_func(q_input, 1, scope="target_q_func", num_units=num_units)[:,0] target_q_func_vars = U.scope_vars(U.absolute_scope_name("target_q_func")) update_target_q = make_update_exp(q_func_vars, target_q_func_vars) target_q_values = U.function(obs_ph_n + act_ph_n, target_q) return train, update_target_q, {'q_values': q_values, 'target_q_values': target_q_values}
def p_train(observPlaceHolderList, actionSpaceList, agentIndex, p_func, q_func, optimizer, grad_norm_clipping, ddpg, num_units=64, scope="trainer", reuse=None): with tf.variable_scope(scope, reuse=reuse): # set up placeholders actionPlaceHolderList = [ tf.placeholder(dtype=tf.float32, shape=[None] + [actionSpaceList[i].n], name="action" + str(i)) for i in range(len(actionSpaceList)) ] policyNetInput = observPlaceHolderList[ agentIndex] # personal observation policyOutputShape = int(actionSpaceList[agentIndex].n) policyTrainOutput = p_func(policyNetInput, policyOutputShape, scope="p_func", num_units=num_units) policyNetVariables = U.scope_vars(U.absolute_scope_name("p_func")) sampleNoise = tf.random_uniform(tf.shape(policyTrainOutput), seed=0) actionSample = U.softmax(policyTrainOutput - tf.log(-tf.log(sampleNoise)), axis=-1) # output of function act p_reg = tf.reduce_mean(tf.square(policyTrainOutput)) actionInputPlaceHolderList = actionPlaceHolderList + [] actionInputPlaceHolderList[agentIndex] = actionSample qNetInput = tf.concat( observPlaceHolderList + actionInputPlaceHolderList, 1) if ddpg: qNetInput = tf.concat( [observPlaceHolderList[agentIndex], actionSample], 1) q = q_func(qNetInput, 1, scope="q_func", reuse=True, num_units=num_units)[:, 0] pg_loss = -tf.reduce_mean(q) loss = pg_loss + p_reg * 1e-3 ####### didnt change this optimization process in my ddpg optimize_expr = U.minimize_and_clip(optimizer, loss, policyNetVariables, grad_norm_clipping) # Create callable functions train = U.function(inputs=observPlaceHolderList + actionPlaceHolderList, outputs=loss, updates=[optimize_expr]) act = U.function(inputs=[observPlaceHolderList[agentIndex]], outputs=actionSample) p_values = U.function([observPlaceHolderList[agentIndex]], policyTrainOutput) # target network target_p = p_func(policyNetInput, int(actionSpaceList[agentIndex].n), scope="target_p_func", num_units=num_units) targetNetVariables = U.scope_vars( U.absolute_scope_name("target_p_func")) update_target_p = make_update_exp(policyNetVariables, targetNetVariables) uTarget = tf.random_uniform(tf.shape(target_p)) target_act_sample = U.softmax(target_p - tf.log(-tf.log(uTarget)), axis=-1) target_act = U.function(inputs=[observPlaceHolderList[agentIndex]], outputs=target_act_sample) return act, train, update_target_p, { 'p_values': p_values, 'target_act': target_act }
def q_train(observPlaceHolderList, actionSpaceList, agentIndex, q_func, optimizer, grad_norm_clipping=None, ddpg=False, scope="trainer", reuse=None, num_units=64): with tf.variable_scope(scope, reuse=reuse): actionPlaceHolderList = [ tf.placeholder(dtype=tf.float32, shape=[None] + [actionSpaceList[i].n], name="action" + str(i)) for i in range(len(actionSpaceList)) ] yi_ = tf.placeholder(tf.float32, [None], name="target") q_input = tf.concat(observPlaceHolderList + actionPlaceHolderList, 1) # shape (?, 24) if ddpg: q_input = tf.concat([ observPlaceHolderList[agentIndex], actionPlaceHolderList[agentIndex] ], 1) # shape (?, 13) q = q_func( q_input, 1, scope="q_func", num_units=num_units)[:, 0] # drop a level: shape (?, 1) to shape (?,) q_func_vars = U.scope_vars(U.absolute_scope_name("q_func")) loss = tf.reduce_mean(tf.square(q - yi_)) optimize_expr = U.minimize_and_clip(optimizer, loss, q_func_vars, grad_norm_clipping) # Create callable functions train = U.function(inputs=observPlaceHolderList + actionPlaceHolderList + [yi_], outputs=loss, updates=[optimize_expr]) q_values = U.function(observPlaceHolderList + actionPlaceHolderList, q) # target network target_q = q_func(q_input, 1, scope="target_q_func", num_units=num_units)[:, 0] target_q_func_vars = U.scope_vars( U.absolute_scope_name("target_q_func")) update_target_q = make_update_exp(q_func_vars, target_q_func_vars) target_q_values = U.function( observPlaceHolderList + actionPlaceHolderList, target_q) return train, update_target_q, { 'q_values': q_values, 'target_q_values': target_q_values }