def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 512):
super(CnnAttn, self).__init__()
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
self.features_dim = features_dim
n_input_channels=observation_space.shape[0]
self.cnn = nn.Sequential(
#nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),
nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=1, padding=0),
nn.ReLU(),
#nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),
nn.Conv2d(32, 64, kernel_size=5, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(64, 64, kernel_size=2, stride=1, padding=0),
nn.ReLU(),
)
# Compute shape by doing one forward pass
with th.no_grad():
in_attn = (self.cnn(th.as_tensor(observation_space.sample()[None])).float()).shape[1]
self.attn = Self_Attn(in_attn)
self.flatten = (nn.Flatten())
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = (self.flatten(self.attn(self.cnn(th.as_tensor(observation_space.sample()[None])).float()))).shape[1]
self.linear = nn.Sequential(nn.Flatten(), nn.Linear(n_flatten, features_dim), nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
linear_dim: int,
features_dim: int = 256,
n_channels: int = 1):
super(CustomCNN, self).__init__(observation_space,
features_dim + linear_dim)
self.linear_dim = linear_dim
self.n_channels = n_channels
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
self.cnn = nn.Sequential(
nn.Conv2d(n_channels, 8, kernel_size=8, stride=4, padding=0),
nn.ReLU(),
nn.Conv2d(8, 16, kernel_size=4, stride=2, padding=0),
nn.ReLU(),
nn.Conv2d(16, 16, kernel_size=3, stride=1, padding=0),
nn.ReLU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with th.no_grad():
obs = th.as_tensor(observation_space.sample()[None]).float()
cam_obs = self._get_camera_obs(obs)
n_flatten = self.cnn(cam_obs).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 512):
super(NatureCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
assert is_image_space(observation_space, check_channels=False), (
"You should use NatureCNN "
f"only with images not with {observation_space}\n"
"(you are probably using `CnnPolicy` instead of `MlpPolicy` or `MultiInputPolicy`)\n"
"If you are using a custom environment,\n"
"please check it using our env checker:\n"
"https://stable-baselines3.readthedocs.io/en/master/common/env_checker.html"
)
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4,
padding=0),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),
nn.ReLU(),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0),
nn.ReLU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 512):
super(NatureCNN, self).__init__(observation_space, features_dim)
# We assume CxWxH images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
assert is_image_space(observation_space), (
'You should use NatureCNN '
f'only with images not with {observation_space} '
'(you are probably using `CnnPolicy` instead of `MlpPolicy`)')
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4,
padding=0), nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0), nn.ReLU(),
nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=0), nn.ReLU(),
nn.Flatten())
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 512):
super(FeatureExtractor, self).__init__(observation_space, features_dim)
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 16, kernel_size=4, stride=1,
padding=0),
nn.ReLU(),
nn.Conv2d(16, 256, kernel_size=32, stride=1, padding=0),
nn.GELU(),
nn.Conv2d(256, 2, kernel_size=2, stride=1, padding=0),
nn.CELU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 256):
super(CustomCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
n_input_channels = observation_space.shape[0]
self.Conv2d1 = nn.Conv2d(n_input_channels,
16,
kernel_size=8,
stride=4,
padding=0)
self.relu1 = nn.ReLU()
self.Conv2d2 = nn.Conv2d(16, 32, kernel_size=8, stride=4, padding=0)
self.relu2 = nn.ReLU()
self.Conv2d3 = nn.Conv2d(32, 32, kernel_size=6, stride=3, padding=0)
self.relu3 = nn.ReLU()
self.Conv2d4 = nn.Conv2d(32, 32, kernel_size=4, stride=2, padding=0)
self.relu4 = nn.ReLU()
self.flatten = nn.Flatten()
# Compute shape by doing one forward pass
with torch.no_grad():
n_flatten = self.forward_cnn(
torch.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear1 = nn.Linear(n_flatten, features_dim)
self.relu5 = nn.ReLU()
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 256):
super(CustomCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 8, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.BatchNorm2d(8),
nn.Conv2d(8, 16, kernel_size=3, stride=1, padding=1),
nn.ReLU(),
nn.BatchNorm2d(16),
nn.Flatten(),
nn.Linear(400, 32, bias=True),
nn.ReLU(),
nn.Linear(32, 16, bias=True),
nn.ReLU(),
nn.Linear(16, 8, bias=True),
)
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self, observation_space: gym.spaces.Box, features_dim: int):
super(ColoringCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
assert is_image_space(observation_space, channels_last=False), (
"You should use NatureCNN "
f"only with images not with {observation_space} "
"(you are probably using `CnnPolicy` instead of `MlpPolicy`)")
in_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(in_channels, 32, kernel_size=1, stride=1),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=3, stride=1),
nn.ReLU(),
nn.Conv2d(64, 64, kernel_size=3, stride=1),
nn.ReLU(),
nn.Conv2d(64, 128, kernel_size=5, stride=1),
nn.ReLU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self, observation_space: gym.spaces.Box, features_dim: int = 256, n_hidden = 1, dropout_rate=0):
super(CustomCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
n_input_channels = observation_space.shape[0]
# print("n_input_channels:", n_input_channels)
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4, padding=0),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),
nn.ReLU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
# print(observation_space.shape)
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(observation_space.sample()[None]).float()
).shape[1]
layers = [nn.Linear(n_flatten, features_dim), nn.ReLU(), nn.Dropout(dropout_rate)]
for i in range(n_hidden - 1):
layers.append(nn.Linear(features_dim, features_dim))
layers.append(nn.ReLU())
layers.append(nn.Dropout(dropout_rate))
self.linear = nn.Sequential(*layers)
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 512):
super(CustomCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
n_input_channels = observation_space.shape[0]
# print(observation_space.shape)
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=8, stride=4,
padding=0),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0),
nn.ReLU(),
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=0),
nn.ReLU(),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with torch.no_grad():
n_flatten = self.cnn(
torch.as_tensor(
observation_space.sample()[None]).float()).shape[1]
print("POST CONV FEATURES = ", n_flatten)
# define the hidden layer to translate to a fixed number of features
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 128):
super(CustomCNN, self).__init__(observation_space, features_dim)
# We assume CxHxW images (channels first)
# Re-ordering will be done by pre-preprocessing or wrapper
"""
extractors = {}
total_concat_size = 0
# We need to know size of the output of this extractor,
# so go over all the spaces and compute output feature sizes
for key, subspace in observation_space.spaces.items():
if key == "image":
# We will just downsample one channel of the image by 4x4 and flatten.
# Assume the image is single-channel (subspace.shape[0] == 0)
extractors[key] = nn.Sequential(nn.MaxPool2d(4), nn.Flatten())
total_concat_size += subspace.shape[1] // 4 * subspace.shape[2] // 4
elif key == "vector":
# Run through a simple MLP
extractors[key] = nn.Linear(subspace.shape[0], 16)
total_concat_size += 16
self.extractors = nn.ModuleDict(extractors)
self._features_dim = total_concat_size
"""
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=6, padding=2),
nn.MaxPool2d(3, 3),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=4), #, stride=2),
nn.MaxPool2d(2, 2),
nn.ReLU(),
nn.Conv2d(64, 128, kernel_size=4),
nn.MaxPool2d(2, 2),
nn.ReLU(),
nn.Conv2d(128, 256, kernel_size=3),
nn.MaxPool2d(2, 2),
nn.ReLU(),
nn.Flatten())
# Compute shape by doing one forward pass
with torch.no_grad():
n_flatten = self.cnn(
torch.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def assert_in_box_space(self, sample: np.array,
space: gym.spaces.Box) -> None:
if space.contains(sample):
return
else:
is_too_low = sample < space.low
is_too_high = sample > space.high
msg = 'Sample is not in space:'
for i in range(len(sample)):
if is_too_low[i]:
msg += f'\nelement {i} too low: {sample[i]} < {space.low[i]}'
if is_too_high[i]:
msg += f'\nelement {i} too high: {sample[i]} > {space.high[i]}'
raise AssertionError(msg)
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 256):
super(CustomCNN, self).__init__(observation_space, features_dim)
n_input_channels = observation_space.shape[0]
self.cnn = nn.Sequential(
nn.Conv2d(n_input_channels, 32, kernel_size=3, stride=1,
padding=0),
nn.ReLU(),
nn.Conv2d(32, 64, kernel_size=2, stride=1, padding=0),
nn.ReLU(),
nn.Flatten(),
)
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, features_dim),
nn.ReLU())
def __init__(self,
observation_space: gym.spaces.Box,
features_dim: int = 128,
**kwargs):
super().__init__(observation_space, features_dim)
channels, height, width = observation_space.shape
self.cnn = nn.Sequential(
nn.LayerNorm([channels, height, width]),
nn.Conv2d(channels,
32,
kernel_size=8,
stride=4,
padding=0,
bias=False),
nn.LayerNorm([
32, 24, 39
]), # TODO: find automatically the weights of the layer norm
nn.LeakyReLU(negative_slope=0.1),
nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=0, bias=False),
nn.LayerNorm([64, 11, 18]),
nn.LeakyReLU(negative_slope=0.1),
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=0, bias=False),
nn.LayerNorm([64, 9, 16]),
nn.LeakyReLU(negative_slope=0.1),
nn.Flatten(),
)
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(
th.as_tensor(
observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(
nn.Linear(n_flatten, features_dim, bias=False),
nn.LayerNorm(features_dim),
nn.LeakyReLU(negative_slope=0.1),
)
def __init__(self, pg_agent_config : PolicyGradientAgentConfig, observation_space: gym.spaces.Box,
features_dim: int = 512, node_net : bool = False, at_net : bool = False):
super(NatureCNN, self).__init__(pg_agent_config, observation_space, pg_agent_config.features_dim,
at_net=at_net, node_net=node_net)
n_input_channels = observation_space.shape[0]
if pg_agent_config.cnn_type == 0:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=2, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=2, out_channels=2, kernel_size=2, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=2, out_channels=2, kernel_size=2, stride=1, padding=0),
nn.ReLU(),
nn.Flatten())
elif pg_agent_config.cnn_type == 1:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=8, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=8, out_channels=8, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=8, out_channels=8, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Flatten())
elif pg_agent_config.cnn_type == 2:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=2, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=2, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Flatten())
elif pg_agent_config.cnn_type == 3:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.MaxPool2d(kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Flatten())
elif pg_agent_config.cnn_type == 4:
self.cnn = IdsGameResNet(in_channels=6, output_dim=44)
elif pg_agent_config.cnn_type == 5:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=64, out_channels=64, kernel_size=1, stride=1, padding=0),
nn.ReLU(),
nn.Flatten())
elif pg_agent_config.cnn_type == 6:
self.cnn = nn.Sequential(nn.Conv2d(n_input_channels, out_channels=2, kernel_size=3, stride=1, padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=2, out_channels=2, kernel_size=3, stride=1,
padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=2, out_channels=2, kernel_size=2, stride=1,
padding=0),
nn.ReLU(),
nn.Conv2d(in_channels=2, out_channels=2, kernel_size=1, stride=1,
padding=0),
nn.ReLU(),
nn.Flatten())
if not self.pg_agent_config.cnn_type == 4:
# Compute shape by doing one forward pass
with th.no_grad():
n_flatten = self.cnn(th.as_tensor(observation_space.sample()[None]).float()).shape[1]
self.linear = nn.Sequential(nn.Linear(n_flatten, pg_agent_config.features_dim), nn.ReLU())
def _(space: gym.spaces.Box) -> Tuple[np.ndarray, np.float32]:
assert space.is_bounded()
sample = np.random.uniform(space.low, space.high,
space.shape).astype(space.dtype)
logprob = -np.log(space.high - space.low).sum().astype(np.float32)
return sample, logprob
