def __init__(self,
env_spec,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
action_merge_layer=-2,
output_nonlinearity=None,
bn=False,
dropout=.05):
Serializable.quick_init(self, locals())
l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim),
name="obs")
l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim),
name="actions")
n_layers = len(hidden_sizes) + 1
if n_layers > 1:
action_merge_layer = \
(action_merge_layer % n_layers + n_layers) % n_layers
else:
action_merge_layer = 1
l_hidden = l_obs
for idx, size in enumerate(hidden_sizes):
if bn:
l_hidden = batch_norm(l_hidden)
if idx == action_merge_layer:
l_hidden = L.ConcatLayer([l_hidden, l_action])
l_hidden = L.DenseLayer(l_hidden,
num_units=size,
nonlinearity=hidden_nonlinearity,
name="h%d" % (idx + 1))
l_hidden = L.DropoutLayer(l_hidden, dropout)
if action_merge_layer == n_layers:
l_hidden = L.ConcatLayer([l_hidden, l_action])
l_output = L.DenseLayer(l_hidden,
num_units=1,
nonlinearity=output_nonlinearity,
name="output")
output_var = L.get_output(l_output, deterministic=True)
output_var_drop = L.get_output(l_output, deterministic=False)
self._f_qval = tensor_utils.compile_function(
[l_obs.input_var, l_action.input_var], output_var)
self._f_qval_drop = tensor_utils.compile_function(
[l_obs.input_var, l_action.input_var], output_var_drop)
self._output_layer = l_output
self._obs_layer = l_obs
self._action_layer = l_action
self._output_nonlinearity = output_nonlinearity
LayersPowered.__init__(self, [l_output])
def __init__(
self,
name,
output_dim,
hidden_sizes,
hidden_nonlinearity,
output_nonlinearity,
dropout_prob=.05,
hidden_W_init=L.XavierUniformInitializer(),
hidden_b_init=tf.zeros_initializer(),
output_W_init=L.XavierUniformInitializer(),
output_b_init=tf.zeros_initializer(),
input_var=None,
input_layer=None,
input_shape=None,
batch_normalization=False,
weight_normalization=False,
):
Serializable.quick_init(self, locals())
with tf.variable_scope(name):
if input_layer is None:
l_in = L.InputLayer(shape=(None, ) + input_shape,
input_var=input_var,
name="input")
else:
l_in = input_layer
self._layers = [l_in]
l_hid = l_in
if batch_normalization:
l_hid = L.batch_norm(l_hid)
for idx, hidden_size in enumerate(hidden_sizes):
l_hid = L.DenseLayer(l_hid,
num_units=hidden_size,
nonlinearity=hidden_nonlinearity,
name="hidden_%d" % idx,
W=hidden_W_init,
b=hidden_b_init,
weight_normalization=weight_normalization)
l_hid = L.DropoutLayer(l_hid, dropout_prob)
self._layers.append(l_hid)
l_out = L.DenseLayer(l_hid,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="output",
W=output_W_init,
b=output_b_init,
weight_normalization=weight_normalization)
if batch_normalization:
l_out = L.batch_norm(l_out)
self._layers.append(l_out)
self._l_in = l_in
self._l_out = l_out
# self._input_var = l_in.input_var
self._output = L.get_output(l_out)
LayersPowered.__init__(self, l_out)
def _make_prop_module(self,
prev_dict: Dict[UnboundAction, Any],
pred_name: str,
output_size: int,
layer_num: int,
l_in: Any,
norm_response: bool=False,
dropout: float=0.0):
# TODO: comment about indexing futzing also applies here---should try
# to do it just once.
name_pfx = 'prop_mod_%s_%d' % (pred_name, layer_num)
prob_meta = self._prob_meta
dom_meta = self._weight_manager.dom_meta
act_to_tensor_idx, prev_inputs = self._sort_inputs(prev_dict)
index_spec = [] # type: List[Tuple[int, List[List[int]]]]
pred_rela = dom_meta.rel_acts(pred_name)
for rel_act_idx, rel_unbound_act in enumerate(pred_rela):
pools = []
for prop in prob_meta.pred_to_props(pred_name):
# we're looking at the act_pred_idx-th relevant proposition
ground_acts = [
ground_act
for unbound_act, ground_acts in prob_meta.rel_acts(prop)
for ground_act in ground_acts
if unbound_act == rel_unbound_act
]
act_inds = [
prob_meta.act_to_schema_subtensor_ind(ground_act)
for ground_act in ground_acts
]
pools.append(act_inds)
tensor_idx = act_to_tensor_idx[rel_unbound_act]
index_spec.append((tensor_idx, pools))
conv_input = PickPoolAndStack(
prev_inputs, index_spec, name=name_pfx + '/cat')
W, b = self._weight_manager.prop_weights[layer_num][pred_name]
rv = L.Conv1DLayer(
conv_input,
output_size,
filter_size=1,
stride=1,
pad='VALID',
W=W,
b=b,
nonlinearity=self.nonlinearity,
name=name_pfx + '/conv')
if dropout > 0:
rv = L.DropoutLayer(rv, p=dropout, name=name_pfx + '/drop')
if norm_response:
rv = ResponseNormLayer(rv, name=name_pfx + '/norm')
# this won't work because batch norm is a mess to apply here
# rv = L.BatchNormLayer(rv, center=True, scale=True)
return rv
def _make_action_module(self,
prev_dict: Dict[str, Any],
unbound_act: UnboundAction,
output_size: int,
layer_num: int,
l_in: L.Layer,
nonlinearity: Any=None,
dropout: float=0.0,
norm_response=False,
extra_dict=None) -> Any:
# TODO: can I do all of this index-futzing just once, instead of each
# time I need to make an action module? Same applies to proposition
# modules. Will make construction much faster (not that it's very
# expensive at the moment...).
name_pfx = 'act_mod_%s_%d' % (unbound_act.schema_name, layer_num)
prob_meta = self._prob_meta
dom_meta = self._weight_manager.dom_meta
# sort input layers so we can pass them to Lasagne
pred_to_tensor_idx, prev_inputs = self._sort_inputs(prev_dict)
# this tells us how many channels our input will have to be
index_spec = []
dom_rel_preds = dom_meta.rel_pred_names(unbound_act)
for act_pred_idx, arg_pred in enumerate(dom_rel_preds):
pools = []
for ground_act in prob_meta.schema_to_acts(unbound_act):
# we're looking at the act_pred_idx-th relevant proposition
bound_prop = prob_meta.rel_props(ground_act)[act_pred_idx]
prop_idx = prob_meta.prop_to_pred_subtensor_ind(bound_prop)
# we're only "pooling" over one element (the proposition
# features)
pools.append([prop_idx])
# which tensor do we need to pick this out of?
tensor_idx = pred_to_tensor_idx[arg_pred]
index_spec.append((tensor_idx, pools))
conv_input = PickPoolAndStack(
prev_inputs, index_spec, name=name_pfx + '/cat')
if layer_num == 0 and extra_dict is not None:
# first action layer, so add in extra data
act_data = extra_dict[unbound_act]
conv_input = L.ConcatLayer(
[conv_input, act_data], axis=2, name=name_pfx + '/extra_cat')
W, b = self._weight_manager.act_weights[layer_num][unbound_act]
if nonlinearity is None:
nonlinearity = self.nonlinearity
rv = L.Conv1DLayer(
conv_input,
output_size,
filter_size=1,
stride=1,
pad='VALID',
W=W,
b=b,
nonlinearity=nonlinearity,
name=name_pfx + '/conv')
if dropout > 0:
rv = L.DropoutLayer(rv, p=dropout, name=name_pfx + '/drop')
if norm_response and output_size > 1:
rv = ResponseNormLayer(rv, name=name_pfx + '/norm')
# BN won't work because it's a mess to apply in a net like this
# rv = L.BatchNormLayer(rv, center=True, scale=True)
return rv
def __init__(
self,
name,
output_dim,
hidden_sizes,
hidden_nonlinearity,
dropout_prob,
output_nonlinearity,
hidden_W_init=L.XavierUniformInitializer(),
hidden_b_init=tf.zeros_initializer(),
output_W_init=L.XavierUniformInitializer(),
output_b_init=tf.zeros_initializer(),
input_var=None,
input_layer=None,
input_shape=None,
batch_normalization=False,
weight_normalization=False,
):
Serializable.quick_init(self, locals())
with tf.variable_scope(name):
if input_layer is None:
l_in = L.InputLayer(shape=(None, ) + input_shape,
input_var=input_var,
name="input")
else:
l_in = input_layer
self._layers = [l_in]
##applying dropout on all layers?
l_hid_dropout_input = L.DropoutLayer(l_in, p=dropout_prob)
l_hid = l_hid_dropout_input
# l_hid = l_in
if batch_normalization:
l_hid = L.batch_norm(l_hid)
for idx, hidden_size in enumerate(hidden_sizes):
l_hid = L.DenseLayer(l_hid,
num_units=hidden_size,
nonlinearity=hidden_nonlinearity,
name="hidden_%d" % idx,
W=hidden_W_init,
b=hidden_b_init,
weight_normalization=weight_normalization)
if batch_normalization:
l_hid = L.batch_norm(l_hid)
self._layers.append(l_hid)
###applying dropout to the last hidden layer?
l_hid_dropout = L.DropoutLayer(l_hid, p=dropout_prob)
l_out = L.DenseLayer(l_hid_dropout,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="output",
W=output_W_init,
b=output_b_init,
weight_normalization=weight_normalization)
# l_out = L.DenseLayer(
# l_hid,
# num_units=output_dim,
# nonlinearity=output_nonlinearity,
# name="output",
# W=output_W_init,
# b=output_b_init,
# weight_normalization=weight_normalization
# )
#Alternative, making output layer the dropout layer
# l_out = L.DropoutLayer(l_hid, p=dropout_prob)
if batch_normalization:
l_out = L.batch_norm(l_out)
self._layers.append(l_out)
self._l_in = l_in
self._l_out = l_out
# self._input_var = l_in.input_var
self._output = L.get_output(l_out)
LayersPowered.__init__(self, l_out)