def build_targetTrain_DQN(make_obs_ph,
                          make_target_ph,
                          q_func,
                          num_actions,
                          optimizer,
                          scope="deepq",
                          reuse=None):

    with tf.variable_scope(scope, reuse=reuse):

        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        target_input = U.ensure_tf_input(make_target_ph("target"))

        # get variables
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))

        # q values for all actions
        q_t_raw = q_func(obs_t_input.get(),
                         num_actions,
                         scope="q_func",
                         reuse=True)

        # calculate error
        td_error = q_t_raw - tf.stop_gradient(target_input.get())
        errors = U.huber_loss(td_error)

        optimize_expr = optimizer.minimize(errors, var_list=q_func_vars)

        targetTrain = U.function(inputs=[obs_t_input, target_input],
                                 outputs=[td_error],
                                 updates=[optimize_expr])

        return targetTrain
Ejemplo n.º 2
0
def build_targetTrain(make_actionDeic_ph,
                        make_target_ph,
                        make_weight_ph,
                        q_func,
                        num_states,
                        num_cascade,
                        optimizer,
                        scope="deepq", 
                        qscope="q_func",
                        grad_norm_clipping=None,
                        reuse=None):

    with tf.variable_scope(scope, reuse=reuse):
        
        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_actionDeic_ph("action_t_deic"))
        target_input = U.ensure_tf_input(make_target_ph("target"))
        importance_weights_ph = U.ensure_tf_input(make_weight_ph("target"))
    
        # get variables
        q_func_vars = U.scope_vars(U.absolute_scope_name(qscope))
    
        # q values for all actions
#        q_t_raw = q_func(obs_t_input.get(), num_states*num_cascade, scope=qscope, reuse=True)
#        targetTiled = tf.reshape(target_input.get(), shape=(-1,num_cascade*num_states))
#        q_t_raw = q_func(obs_t_input.get(), num_states, scope=qscope, reuse=True)
#        targetTiled = tf.reshape(target_input.get(), shape=(-1,num_states))
        q_t_raw = q_func(obs_t_input.get(), 1, scope=qscope, reuse=True)
        targetTiled = tf.reshape(target_input.get(), shape=(-1,1))
        
        # calculate error
        td_error = q_t_raw - tf.stop_gradient(targetTiled)
        errors = importance_weights_ph.get() * U.huber_loss(td_error)

        # compute optimization op (potentially with gradient clipping)
        if grad_norm_clipping is not None:
            optimize_expr = U.minimize_and_clip(optimizer,
                                                errors,
                                                var_list=q_func_vars,
                                                clip_val=grad_norm_clipping)
        else:
            optimize_expr = optimizer.minimize(errors, var_list=q_func_vars)
    
#        optimize_expr = optimizer.minimize(errors, var_list=q_func_vars)
        
        targetTrain = U.function(
            inputs=[
                obs_t_input,
                target_input,
                importance_weights_ph
            ],
            outputs=[td_error, obs_t_input.get(), target_input.get()],
            updates=[optimize_expr]
        )
    
        return targetTrain
def build_train_cascaded(make_obs_ph,
                         make_target_ph,
                         q_func,
                         num_cascade,
                         num_actions,
                         optimizer,
                         grad_norm_clipping=None,
                         double_q=True,
                         scope="deepq",
                         reuse=None):

    getq_f = build_getq(make_obs_ph,
                        q_func,
                        num_actions,
                        num_cascade,
                        scope=scope,
                        reuse=reuse)

    with tf.variable_scope(scope, reuse=reuse):

        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        target_input = U.ensure_tf_input(make_target_ph("target"))

        # get variables
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))

        # q values for all actions
        q_t_raw = q_func(obs_t_input.get(),
                         num_actions * num_cascade,
                         scope="q_func",
                         reuse=True)
        q_t = tf.reshape(q_t_raw, shape=(-1, num_cascade, num_actions))

        # calculate error
        td_error = q_t - tf.stop_gradient(target_input.get())
        errors = U.huber_loss(td_error)

        optimize_expr = optimizer.minimize(errors, var_list=q_func_vars)

        targetTrain = U.function(
            inputs=[obs_t_input, target_input],
            outputs=[td_error, obs_t_input.get(),
                     target_input.get()],
            updates=[optimize_expr])

    return getq_f, targetTrain
Ejemplo n.º 4
0
def build_train_cascaded(make_obs_ph,
                         make_target_ph,
                         make_actions_ph,
                         q_func,
                         num_cascade,
                         num_actions,
                         batch_size,
                         num_deictic_patches,
                         optimizer,
                         gamma=1.0,
                         grad_norm_clipping=None,
                         double_q=True,
                         scope="deepq",
                         reuse=None):

    getq_f = build_getq(make_obs_ph,
                        q_func,
                        num_actions * num_cascade,
                        scope=scope,
                        scope_q_func="q_func",
                        reuse=reuse)
    #    getq_f_target = build_getq(make_obs_ph, q_func, num_actions * num_cascade, scope=scope, scope_q_func="target_q_func", reuse=reuse)

    with tf.variable_scope(scope, reuse=reuse):

        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        actions_input = U.ensure_tf_input(make_actions_ph("actions"))
        target_input = U.ensure_tf_input(make_target_ph("target"))

        # get variables
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))
        target_q_func_vars = U.scope_vars(
            U.absolute_scope_name("target_q_func"))

        # q values for all actions
        q_t_raw = q_func(obs_t_input.get(),
                         num_actions * num_cascade,
                         scope="q_func",
                         reuse=True)  # reuse parameters from act
        q_t = tf.reshape(
            q_t_raw,
            [batch_size * num_deictic_patches, num_cascade, num_actions])

        # q values for selected actions
        actionsTiled = tf.one_hot(actions_input.get(), num_actions)
        q_t_action_select = tf.reduce_sum(q_t * actionsTiled, 2)

        # calculate error
        td_error = q_t_action_select - tf.stop_gradient(target_input.get())
        errors = U.huber_loss(td_error)

        # compute optimization op (potentially with gradient clipping)
        if grad_norm_clipping is not None:
            optimize_expr = U.minimize_and_clip(optimizer,
                                                errors,
                                                var_list=q_func_vars,
                                                clip_val=grad_norm_clipping)
        else:
            optimize_expr = optimizer.minimize(errors, var_list=q_func_vars)

#        # update_target_fn will be called periodically to copy Q network to target Q network
#        update_target_expr = []
#        for var, var_target in zip(sorted(q_func_vars, key=lambda v: v.name),
#                                   sorted(target_q_func_vars, key=lambda v: v.name)):
#            update_target_expr.append(var_target.assign(var))
#        update_target_expr = tf.group(*update_target_expr)

# Create callable functions
        targetTrain = U.function(
            inputs=[obs_t_input, actions_input, target_input],
            outputs=[td_error, q_t_action_select,
                     target_input.get()],
            updates=[optimize_expr])

#        update_target = U.function([], [], updates=[update_target_expr])

#    return getq_f, getq_f_target, targetTrain, update_target
    return getq_f, targetTrain
Ejemplo n.º 5
0
def build_train_deictic_min_streamlined(make_obs_ph,
                                        q_func,
                                        num_actions,
                                        batch_size,
                                        num_deictic_patches,
                                        max_num_groups,
                                        optimizer,
                                        gamma=1.0,
                                        grad_norm_clipping=None,
                                        double_q=True,
                                        scope="deepq",
                                        reuse=None):

    #    act_f = build_act(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse)
    getq_f = build_getq(make_obs_ph,
                        q_func,
                        num_actions,
                        scope=scope,
                        reuse=reuse)

    with tf.variable_scope(scope, reuse=reuse):

        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        act_t_ph = tf.placeholder(tf.int32, [None], name="action")
        obs_tp1_input = U.ensure_tf_input(make_obs_ph("obs_tp1"))
        rew_t_ph = tf.placeholder(tf.float32, [None], name="reward")
        group_matching_ph = tf.placeholder(tf.int32, [None],
                                           name="group_matching")
        done_mask_ph = tf.placeholder(tf.float32, [None], name="done_mask")

        # Creating this placeholder to enable a tabular version of this code
        q_tp1_ph = tf.placeholder(tf.float32, [None, 4], name="q_tp1")

        # q network evaluation
        q_t = q_func(obs_t_input.get(),
                     num_actions,
                     scope="q_func",
                     reuse=True)  # reuse parameters from act
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))

        # target q network evalution
        #        q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope="target_q_func")
        q_tp1 = q_tp1_ph
        target_q_func_vars = U.scope_vars(
            U.absolute_scope_name("target_q_func"))

        # compute estimate of best possible value starting from state at t + 1
        if double_q:
            q_tp1_using_online_net = q_func(obs_tp1_input.get(),
                                            num_actions,
                                            scope="q_func",
                                            reuse=True)
            q_tp1_best_using_online_net = tf.arg_max(q_tp1_using_online_net, 1)
            q_tp1_best = tf.reduce_sum(
                q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions),
                1)
        else:
            q_tp1_maxa = tf.reduce_max(q_tp1, 1)

        # Calculate target = max_{a,d} Q(d,a). Size should be: Bx1
        q_tp1_maxa_reshape = tf.reshape(q_tp1_maxa,
                                        [batch_size, num_deictic_patches])
        q_tp1_target = tf.reduce_max(q_tp1_maxa_reshape, 1)
        q_tp1_target = rew_t_ph + gamma * q_tp1_target
        q_tp1_target = (1.0 - done_mask_ph) * q_tp1_target

        # Calculate desc_2_state. Encodes which descriptors contained in each state.
        # Dimensions should be: B x max_num_groups
        group_matching_3dtensor = tf.reshape(group_matching_ph,
                                             [batch_size, num_deictic_patches])
        groups_onehot = tf.one_hot(group_matching_3dtensor,
                                   max_num_groups,
                                   axis=-1)
        desc_2_state = tf.reduce_max(groups_onehot, 1)

        # Calculate target_min_per_D
        max_target = tf.reduce_max(q_tp1_target)
        q_tp1_target_tiled = tf.tile(tf.reshape(q_tp1_target, [batch_size, 1]),
                                     [1, max_num_groups])
        target_min_per_D_expanded = desc_2_state * q_tp1_target_tiled + (
            1 - desc_2_state) * max_target
        target_min_per_D = tf.reduce_min(target_min_per_D_expanded, 0)

        # Project target_min_per_D onto q_t
        D_2_DI = tf.one_hot(group_matching_ph, max_num_groups)

        # Both the next two lines produce the same results
        targets = tf.squeeze(
            tf.matmul(D_2_DI, tf.reshape(target_min_per_D,
                                         [max_num_groups, 1])))
        #        targets2 = tf.reduce_sum(D_2_DI * tf.tile(tf.reshape(target_min_per_D,[1,max_num_groups]),[batch_size*num_deictic_patches,1]),1)

        #        tf.tile(tf.reshape(target_min_per_D,[max_num_groups,1]),[1,np.shape(q_t)[0]])

        #        # expand target to BxD(I')
        #        q_tp1_target_under_tiled = tf.tile(tf.reshape(q_tp1_target_under,[batch_size,1]),[1,num_deictic_patches])
        #        q_tp1_target = tf.reshape(q_tp1_target_under_tiled,[batch_size*num_deictic_patches,1])
        #        # Calculate min over groups
        #        groups_onehot = tf.one_hot(group_matching_ph,max_num_groups+1)
        #        q_tp1_target_tiled = tf.tile(q_tp1_target,[1,max_num_groups+1])
        #        max_target = tf.reduce_max(q_tp1_target)
        #        groups_target_tiled = groups_onehot*q_tp1_target_tiled + (1-groups_onehot)*max_target
        #        groups_target = tf.reduce_min(groups_target_tiled,0)[:-1]
        #        groups_target_reduced = groups_target[0:tf.minimum(tf.shape(group_actions_ph)[0],tf.shape(groups_target)[0])]

        # q scores for actions which we know were selected in the given state.
        q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions),
                                     1)

        # compute the error (potentially clipped)
        #        td_error = q_group_selected - tf.stop_gradient(groups_target_reduced)
        td_error = q_t_selected - tf.stop_gradient(targets)
        errors = U.huber_loss(td_error)
        weighted_error = errors

        # compute optimization op (potentially with gradient clipping)
        if grad_norm_clipping is not None:
            optimize_expr = U.minimize_and_clip(optimizer,
                                                weighted_error,
                                                var_list=q_func_vars,
                                                clip_val=grad_norm_clipping)
        else:
            optimize_expr = optimizer.minimize(weighted_error,
                                               var_list=q_func_vars)

        # update_target_fn will be called periodically to copy Q network to target Q network
        update_target_expr = []
        for var, var_target in zip(
                sorted(q_func_vars, key=lambda v: v.name),
                sorted(target_q_func_vars, key=lambda v: v.name)):
            update_target_expr.append(var_target.assign(var))
        update_target_expr = tf.group(*update_target_expr)

        # Create callable functions
        train = U.function(inputs=[
            obs_t_input, act_t_ph, rew_t_ph, obs_tp1_input, group_matching_ph,
            done_mask_ph
        ],
                           outputs=[targets],
                           updates=[optimize_expr])

        # Create callable functions
        trainWOUpdate = U.function(inputs=[
            obs_t_input, act_t_ph, rew_t_ph, obs_tp1_input, group_matching_ph,
            done_mask_ph, q_tp1_ph
        ],
                                   outputs=[
                                       q_tp1_target, desc_2_state,
                                       target_min_per_D, D_2_DI, targets
                                   ])

        update_target = U.function([], [], updates=[update_target_expr])
        #        q_values = U.function([obs_t_input], q_t)

        #        return getq_f, train, trainWOUpdate, update_target, {'q_values': q_values}
        return getq_f, train, trainWOUpdate, update_target
Ejemplo n.º 6
0
def build_train(make_obs_ph,
                q_func,
                num_actions,
                optimizer,
                grad_norm_clipping=None,
                gamma=1.0,
                double_q=True,
                scope="deepq",
                reuse=None):
    """Creates the train function:

    Parameters
    ----------
    make_obs_ph: str -> tf.placeholder or TfInput
        a function that takes a name and creates a placeholder of input with that name
    q_func: (tf.Variable, int, str, bool) -> tf.Variable
        the model that takes the following inputs:
            observation_in: object
                the output of observation placeholder
            num_actions: int
                number of actions
            scope: str
            reuse: bool
                should be passed to outer variable scope
        and returns a tensor of shape (batch_size, num_actions) with values of every action.
    num_actions: int
        number of actions
    reuse: bool
        whether or not to reuse the graph variables
    optimizer: tf.train.Optimizer
        optimizer to use for the Q-learning objective.
    grad_norm_clipping: float or None
        clip gradient norms to this value. If None no clipping is performed.
    gamma: float
        discount rate.
    double_q: bool
        if true will use Double Q Learning (https://arxiv.org/abs/1509.06461).
        In general it is a good idea to keep it enabled.
    scope: str or VariableScope
        optional scope for variable_scope.
    reuse: bool or None
        whether or not the variables should be reused. To be able to reuse the scope must be given.

    Returns
    -------
    act: (tf.Variable, bool, float) -> tf.Variable
        function to select and action given observation.
`       See the top of the file for details.
    train: (object, np.array, np.array, object, np.array, np.array) -> np.array
        optimize the error in Bellman's equation.
`       See the top of the file for details.
    update_target: () -> ()
        copy the parameters from optimized Q function to the target Q function.
`       See the top of the file for details.
    debug: {str: function}
        a bunch of functions to print debug data like q_values.
    """
    act_f = build_act(make_obs_ph,
                      q_func,
                      num_actions,
                      scope=scope,
                      reuse=reuse)

    with tf.variable_scope(scope, reuse=reuse):
        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        act_t_ph = tf.placeholder(tf.int32, [None], name="action")
        rew_t_ph = tf.placeholder(tf.float32, [None], name="reward")
        obs_tp1_input = U.ensure_tf_input(make_obs_ph("obs_tp1"))
        done_mask_ph = tf.placeholder(tf.float32, [None], name="done")
        importance_weights_ph = tf.placeholder(tf.float32, [None],
                                               name="weight")

        # q network evaluation
        q_t = q_func(obs_t_input.get(),
                     num_actions,
                     scope="q_func",
                     reuse=True)  # reuse parameters from act
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))

        # target q network evalution
        q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope="target_q_func")
        target_q_func_vars = U.scope_vars(
            U.absolute_scope_name("target_q_func"))

        # q scores for actions which we know were selected in the given state.
        q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions),
                                     1)

        # compute estimate of best possible value starting from state at t + 1
        if double_q:
            q_tp1_using_online_net = q_func(obs_tp1_input.get(),
                                            num_actions,
                                            scope="q_func",
                                            reuse=True)
            q_tp1_best_using_online_net = tf.arg_max(q_tp1_using_online_net, 1)
            q_tp1_best = tf.reduce_sum(
                q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions),
                1)
        else:
            q_tp1_best = tf.reduce_max(q_tp1, 1)
        q_tp1_best_masked = (1.0 - done_mask_ph) * q_tp1_best

        # compute RHS of bellman equation
        q_t_selected_target = rew_t_ph + gamma * q_tp1_best_masked

        # compute the error (potentially clipped)
        td_error = q_t_selected - tf.stop_gradient(q_t_selected_target)
        errors = U.huber_loss(td_error)
        weighted_error = tf.reduce_mean(importance_weights_ph * errors)
        # compute optimization op (potentially with gradient clipping)
        if grad_norm_clipping is not None:
            optimize_expr = U.minimize_and_clip(optimizer,
                                                weighted_error,
                                                var_list=q_func_vars,
                                                clip_val=grad_norm_clipping)
        else:
            optimize_expr = optimizer.minimize(weighted_error,
                                               var_list=q_func_vars)

        # update_target_fn will be called periodically to copy Q network to target Q network
        update_target_expr = []
        for var, var_target in zip(
                sorted(q_func_vars, key=lambda v: v.name),
                sorted(target_q_func_vars, key=lambda v: v.name)):
            update_target_expr.append(var_target.assign(var))
        update_target_expr = tf.group(*update_target_expr)

        # Create callable functions
        train = U.function(inputs=[
            obs_t_input, act_t_ph, rew_t_ph, obs_tp1_input, done_mask_ph,
            importance_weights_ph
        ],
                           outputs=td_error,
                           updates=[optimize_expr])
        update_target = U.function([], [], updates=[update_target_expr])

        q_values = U.function([obs_t_input], q_t)

        return act_f, train, update_target, {'q_values': q_values}
Ejemplo n.º 7
0
def build_train_deictic(make_obs_ph,
                        q_func,
                        num_actions,
                        optimizer,
                        grad_norm_clipping=None,
                        gamma=1.0,
                        double_q=True,
                        scope="deepq",
                        reuse=None):

    #    act_f = build_act(make_obs_ph, q_func, num_actions, scope=scope, reuse=reuse)
    getq_f = build_getq(make_obs_ph,
                        q_func,
                        num_actions,
                        scope=scope,
                        reuse=reuse)

    with tf.variable_scope(scope, reuse=reuse):
        # set up placeholders
        obs_t_input = U.ensure_tf_input(make_obs_ph("obs_t"))
        act_t_ph = tf.placeholder(tf.int32, [None], name="action")
        rew_t_ph = tf.placeholder(tf.float32, [None], name="reward")
        obs_tp1_input = U.ensure_tf_input(make_obs_ph("obs_tp1"))
        done_mask_ph = tf.placeholder(tf.float32, [None], name="done")
        importance_weights_ph = tf.placeholder(tf.float32, [None],
                                               name="weight")

        # q network evaluation
        q_t = q_func(obs_t_input.get(),
                     num_actions,
                     scope="q_func",
                     reuse=True)  # reuse parameters from act
        q_func_vars = U.scope_vars(U.absolute_scope_name("q_func"))

        # target q network evalution
        q_tp1 = q_func(obs_tp1_input.get(), num_actions, scope="target_q_func")
        target_q_func_vars = U.scope_vars(
            U.absolute_scope_name("target_q_func"))

        # q scores for actions which we know were selected in the given state.
        q_t_selected = tf.reduce_sum(q_t * tf.one_hot(act_t_ph, num_actions),
                                     1)

        # compute estimate of best possible value starting from state at t + 1
        if double_q:
            q_tp1_using_online_net = q_func(obs_tp1_input.get(),
                                            num_actions,
                                            scope="q_func",
                                            reuse=True)
            q_tp1_best_using_online_net = tf.arg_max(q_tp1_using_online_net, 1)
            q_tp1_best = tf.reduce_sum(
                q_tp1 * tf.one_hot(q_tp1_best_using_online_net, num_actions),
                1)
        else:
            q_tp1_best = tf.reduce_max(q_tp1, 1)

        q_tp1_best = tf.reshape(q_tp1_best, [32, 25])
        q_tp1_best_reduced = tf.reduce_max(q_tp1_best, 1)

        # compute RHS of bellman equation
        q_t_selected_target = rew_t_ph + gamma * q_tp1_best_reduced
        q_t_selected_target_masked = (1.0 - done_mask_ph) * q_t_selected_target

        q_t_selected_target_tiled = tf.tile(
            tf.reshape(q_t_selected_target_masked, [32, 1]), [1, 25])
        q_t_selected_target_expanded = tf.reshape(q_t_selected_target_tiled, [
            800,
        ])

        # compute the error (potentially clipped)
        td_error = q_t_selected - tf.stop_gradient(
            q_t_selected_target_expanded)
        errors = U.huber_loss(td_error)
        weighted_error = tf.reduce_mean(importance_weights_ph * errors)

        # compute optimization op (potentially with gradient clipping)
        if grad_norm_clipping is not None:
            optimize_expr = U.minimize_and_clip(optimizer,
                                                weighted_error,
                                                var_list=q_func_vars,
                                                clip_val=grad_norm_clipping)
        else:
            optimize_expr = optimizer.minimize(weighted_error,
                                               var_list=q_func_vars)

        # update_target_fn will be called periodically to copy Q network to target Q network
        update_target_expr = []
        for var, var_target in zip(
                sorted(q_func_vars, key=lambda v: v.name),
                sorted(target_q_func_vars, key=lambda v: v.name)):
            update_target_expr.append(var_target.assign(var))
        update_target_expr = tf.group(*update_target_expr)

        # Create callable functions
        train = U.function(inputs=[
            obs_t_input, act_t_ph, rew_t_ph, obs_tp1_input, done_mask_ph,
            importance_weights_ph
        ],
                           outputs=td_error,
                           updates=[optimize_expr])

        # Create callable functions
        trainWOUpdate = U.function(
            inputs=[
                obs_t_input, act_t_ph, rew_t_ph, obs_tp1_input, done_mask_ph,
                importance_weights_ph
            ],
            outputs=[q_t_selected_target_expanded, errors, td_error])

        update_target = U.function([], [], updates=[update_target_expr])

        q_values = U.function([obs_t_input], q_t)

        #        return act_f, train, update_target, {'q_values': q_values}
        #        return getq_f, train, trainWOUpdate, update_target, {'q_values': q_values}
        #        return getq_f, train, trainWOUpdate, {'q_values': q_values}
        return getq_f, train, trainWOUpdate, update_target, {
            'q_values': q_values
        }