def inputs(self):
return [
TensorSpec([None] + list(self._image_shape),
tf.float32,
name='input'),
TensorSpec([None], tf.int32, name='class')
]
def _encode_numerical_feature(
feature: KerasTensor,
name: str,
dataset: Optional[BatchDataset],
) -> KerasTensor:
"""Normalize numerical features.
Args:
- feature: The input layer of the feature.
- name: The feature's name (its column name in the original dataframe).
- dataset: The training data, if not specified, return a no-op layer.
Returns:
The normalized tensor of the input feature.
"""
# Return generic layer for the tuner initialization
if not dataset:
return KerasTensor(type_spec=TensorSpec(
shape=(None, 1), dtype=tf.float32, name=None))
# Create a Normalization layer for our feature
normalizer = Normalization()
# Prepare a Dataset that only yields our feature
feature_ds = dataset.map(lambda x, y: x[name])
feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))
# Learn the statistics of the data
normalizer.adapt(feature_ds)
# Normalize the input feature
encoded_feature = normalizer(feature)
return encoded_feature
def _encode_categorical_feature(
feature: KerasTensor,
name: str,
dataset: Optional[BatchDataset],
) -> KerasTensor:
"""One-hot encode categorical features.
Args:
- feature: The input layer of the feature.
- name: The feature's name (its column name in the original dataframe).
- dataset: The training data, if not specified, return a no-op layer.
Returns:
The one-hot encoded tensor of the input feature.
"""
# Return generic layer for the tuner initialization
if not dataset:
return KerasTensor(type_spec=TensorSpec(
shape=(None, 1), dtype=tf.float32, name=None))
# Create a StringLookup layer which will turn strings into integer indices
index = StringLookup()
# Prepare a Dataset that only yields our feature
feature_ds = dataset.map(lambda x, y: x[name])
feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))
# Learn the set of possible string values and assign them a fixed integer index
index.adapt(feature_ds)
# Turn the string input into integer indices
encoded_feature = index(feature)
# Create a CategoryEncoding for our integer indices
encoder = CategoryEncoding(output_mode="binary")
# Learn the space of possible indices
encoder.adapt(np.arange(index.vocab_size()))
# Apply one-hot encoding to our indices{split + 1} / {n_splits}
encoded_feature = encoder(encoded_feature)
return encoded_feature
def observation_spec(self):
return TensorSpec(shape=self.env.observation_space.shape,
dtype=self.env.observation_space.dtype)
def __init__(
self,
root_dir,
conv_1d_layer_params=[(32, 8, 4), (64, 4, 2), (64, 3, 1)],
conv_2d_layer_params=[(32, (8, 8), 4), (64, (4, 4), 2),
(64, (3, 3), 2)],
encoder_fc_layers=[256],
actor_fc_layers=[256],
critic_obs_fc_layers=[256],
critic_action_fc_layers=[256],
critic_joint_fc_layers=[256],
# Params for target update
target_update_tau=0.005,
target_update_period=1,
# Params for train
actor_learning_rate=3e-4,
critic_learning_rate=3e-4,
alpha_learning_rate=3e-4,
td_errors_loss_fn=tf.compat.v1.losses.mean_squared_error,
gamma=0.99,
reward_scale_factor=1.0,
gradient_clipping=None,
# Params for eval
eval_deterministic=False,
# Params for summaries and logging
debug_summaries=False,
summarize_grads_and_vars=False):
'''A simple train and eval for SAC.'''
tf.compat.v1.enable_resource_variables()
root_dir = os.path.expanduser(root_dir)
policy_dir = os.path.join(root_dir, 'train', 'policy')
time_step_spec = TimeStep(
TensorSpec(shape=(), dtype=tf.int32, name='step_type'),
TensorSpec(shape=(), dtype=tf.float32, name='reward'),
BoundedTensorSpec(shape=(),
dtype=tf.float32,
name='discount',
minimum=np.array(0., dtype=np.float32),
maximum=np.array(1., dtype=np.float32)),
collections.OrderedDict({
'task_obs':
BoundedTensorSpec(shape=(TASK_OBS_DIM, ),
dtype=tf.float32,
name=None,
minimum=np.array(-3.4028235e+38,
dtype=np.float32),
maximum=np.array(3.4028235e+38,
dtype=np.float32)),
'depth':
BoundedTensorSpec(shape=(IMG_HEIGHT, IMG_WIDTH, 1),
dtype=tf.float32,
name=None,
minimum=np.array(-1.0, dtype=np.float32),
maximum=np.array(1.0, dtype=np.float32)),
'rgb':
BoundedTensorSpec(shape=(IMG_HEIGHT, IMG_WIDTH, 3),
dtype=tf.float32,
name=None,
minimum=np.array(-1.0, dtype=np.float32),
maximum=np.array(1.0, dtype=np.float32)),
}))
observation_spec = time_step_spec.observation
action_spec = BoundedTensorSpec(shape=(2, ),
dtype=tf.float32,
name=None,
minimum=np.array(-1.0,
dtype=np.float32),
maximum=np.array(1.0,
dtype=np.float32))
glorot_uniform_initializer = tf.compat.v1.keras.initializers.glorot_uniform(
)
preprocessing_layers = {}
if 'rgb' in observation_spec:
preprocessing_layers['rgb'] = tf.keras.Sequential(
mlp_layers(
conv_1d_layer_params=None,
conv_2d_layer_params=conv_2d_layer_params,
fc_layer_params=encoder_fc_layers,
kernel_initializer=glorot_uniform_initializer,
))
if 'depth' in observation_spec:
preprocessing_layers['depth'] = tf.keras.Sequential(
mlp_layers(
conv_1d_layer_params=None,
conv_2d_layer_params=conv_2d_layer_params,
fc_layer_params=encoder_fc_layers,
kernel_initializer=glorot_uniform_initializer,
))
if 'task_obs' in observation_spec:
preprocessing_layers['task_obs'] = tf.keras.Sequential(
mlp_layers(
conv_1d_layer_params=None,
conv_2d_layer_params=None,
fc_layer_params=encoder_fc_layers,
kernel_initializer=glorot_uniform_initializer,
))
if len(preprocessing_layers) <= 1:
preprocessing_combiner = None
else:
preprocessing_combiner = tf.keras.layers.Concatenate(axis=-1)
actor_net = actor_distribution_network.ActorDistributionNetwork(
observation_spec,
action_spec,
preprocessing_layers=preprocessing_layers,
preprocessing_combiner=preprocessing_combiner,
fc_layer_params=actor_fc_layers,
continuous_projection_net=normal_projection_net,
kernel_initializer=glorot_uniform_initializer,
)
critic_net = critic_network.CriticNetwork(
(observation_spec, action_spec),
preprocessing_layers=preprocessing_layers,
preprocessing_combiner=preprocessing_combiner,
observation_fc_layer_params=critic_obs_fc_layers,
action_fc_layer_params=critic_action_fc_layers,
joint_fc_layer_params=critic_joint_fc_layers,
kernel_initializer=glorot_uniform_initializer,
)
global_step = tf.compat.v1.train.get_or_create_global_step()
tf_agent = sac_agent.SacAgent(
time_step_spec,
action_spec,
actor_network=actor_net,
critic_network=critic_net,
actor_optimizer=tf.compat.v1.train.AdamOptimizer(
learning_rate=actor_learning_rate),
critic_optimizer=tf.compat.v1.train.AdamOptimizer(
learning_rate=critic_learning_rate),
alpha_optimizer=tf.compat.v1.train.AdamOptimizer(
learning_rate=alpha_learning_rate),
target_update_tau=target_update_tau,
target_update_period=target_update_period,
td_errors_loss_fn=td_errors_loss_fn,
gamma=gamma,
reward_scale_factor=reward_scale_factor,
gradient_clipping=gradient_clipping,
debug_summaries=debug_summaries,
summarize_grads_and_vars=summarize_grads_and_vars,
train_step_counter=global_step)
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.compat.v1.Session(config=config)
if eval_deterministic:
self.eval_py_policy = py_tf_policy.PyTFPolicy(
greedy_policy.GreedyPolicy(tf_agent.policy))
else:
self.eval_py_policy = py_tf_policy.PyTFPolicy(tf_agent.policy)
policy_checkpointer = common.Checkpointer(ckpt_dir=policy_dir,
policy=tf_agent.policy,
global_step=global_step)
with self.sess.as_default():
# Initialize graph.
policy_checkpointer.initialize_or_restore(self.sess)
# activate the session
obs = {
'depth': np.ones((IMG_HEIGHT, IMG_WIDTH, 1)),
'rgb': np.ones((IMG_HEIGHT, IMG_WIDTH, 3)),
'task_obs': np.ones((TASK_OBS_DIM, ))
}
action = self.act(obs)
print('activate TF session')
print('action', action)
def __init__(self, layer=None, **kwargs):
super(Mobilenet, self).__init__(**kwargs)
self._path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'mobilenet_v1_1.0_224_frozen.pb')
tpl = "MobilenetV1/MobilenetV1/{}_pointwise/Relu6"
self._layer_spec = [
TensorSpec((224, 224, 3), tf.float32, name='input'),
TensorSpec((112, 112, 64), tf.float32,
name=tpl.format("Conv2d_1")),
TensorSpec((56, 56, 128), tf.float32, name=tpl.format("Conv2d_2")),
TensorSpec((56, 56, 128), tf.float32, name=tpl.format("Conv2d_3")),
TensorSpec((28, 28, 256), tf.float32, name=tpl.format("Conv2d_4")),
TensorSpec((28, 28, 256), tf.float32, name=tpl.format("Conv2d_5")),
TensorSpec((14, 14, 512), tf.float32, name=tpl.format("Conv2d_6")),
TensorSpec((14, 14, 512), tf.float32, name=tpl.format("Conv2d_7")),
TensorSpec((14, 14, 512), tf.float32, name=tpl.format("Conv2d_8")),
TensorSpec((14, 14, 512), tf.float32, name=tpl.format("Conv2d_9")),
TensorSpec((14, 14, 512), tf.float32,
name=tpl.format("Conv2d_10")),
TensorSpec((14, 14, 512), tf.float32,
name=tpl.format("Conv2d_11")),
TensorSpec((7, 7, 1024), tf.float32, name=tpl.format("Conv2d_12")),
TensorSpec((7, 7, 1024), tf.float32, name=tpl.format("Conv2d_13"))
]
if layer is None:
layer = -1
if layer < 0:
layer = len(self._layer_spec) + layer
self._output_layer = layer
self._graph_def = None
self._input_shape = (224, 224, 3)
self._output_shape = tuple(self._layer_spec[self._output_layer].shape)