def __init__( self, observation_space, g_action_space, l_action_space, pretrain_path, output_size=512, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__( observation_space, g_action_space, l_action_space, pretrain_path, output_size=512, obs_transform=ResizeCenterCropper(size=(256, 256)), ) self.net = ObjectNavGraphSLAMNet( observation_space=observation_space, g_action_space=g_action_space, output_size=output_size, obs_transform=obs_transform, pretrain_path=pretrain_path, )
def __init__( self, observation_space, action_space, hidden_size=512, num_recurrent_layers=2, rnn_type="LSTM", resnet_baseplanes=32, backbone="resnet50", normalize_visual_inputs=False, obs_transform=ResizeCenterCropper(size=(256, 256)), force_blind_policy=False, ): super().__init__( PointNavResNetNet( observation_space=observation_space, action_space=action_space, hidden_size=hidden_size, num_recurrent_layers=num_recurrent_layers, rnn_type=rnn_type, backbone=backbone, resnet_baseplanes=resnet_baseplanes, normalize_visual_inputs=normalize_visual_inputs, obs_transform=obs_transform, force_blind_policy=force_blind_policy, ), action_space.n, )
def __init__( self, observation_space, action_space, goal_sensor_uuid="pointgoal_with_gps_compass", hidden_size=512, num_recurrent_layers=2, rnn_type="LSTM", resnet_baseplanes=32, backbone="resnet50", normalize_visual_inputs=False, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__( PointNavResNetNet( observation_space=observation_space, action_space=action_space, goal_sensor_uuid=goal_sensor_uuid, hidden_size=hidden_size, num_recurrent_layers=num_recurrent_layers, rnn_type=rnn_type, backbone=backbone, resnet_baseplanes=resnet_baseplanes, normalize_visual_inputs=normalize_visual_inputs, obs_transform=obs_transform, ), action_space.n, )
def __init__( self, observation_space, hidden_size, goal_sensor_uuid=None, detach=False, additional_sensors=[] # low dim sensors corresponding to registered name ): super().__init__() self.goal_sensor_uuid = goal_sensor_uuid self.additional_sensors = additional_sensors self._n_input_goal = 0 self._n_input_goal = 0 # if goal_sensor_uuid is not None and goal_sensor_uuid != "no_sensor": # self.goal_sensor_uuid = goal_sensor_uuid # self._initialize_goal_encoder(observation_space) self._hidden_size = hidden_size resnet_baseplanes = 32 backbone="resnet18" visual_resnet = ResNetEncoder( observation_space, baseplanes=resnet_baseplanes, ngroups=resnet_baseplanes // 2, make_backbone=getattr(resnet, backbone), normalize_visual_inputs=False, obs_transform=ResizeCenterCropper(size=(256, 256)), backbone_only=True, ) self.detach = detach self.visual_resnet = visual_resnet self.visual_encoder = nn.Sequential( Flatten(), nn.Linear( np.prod(visual_resnet.output_shape), hidden_size ), nn.Sigmoid() ) self.visual_decoder = nn.Sequential( nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Upsample(scale_factor=2), nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Upsample(scale_factor=2), nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Upsample(scale_factor=2), nn.Conv2d(64, 32, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Upsample(scale_factor=2), nn.Conv2d(32, 32, kernel_size=3, stride=1, padding=1), nn.ReLU(), nn.Upsample(scale_factor=2), nn.Conv2d(32, 3, kernel_size=3, stride=1, padding=1), ) self.train()
def __init__( self, observation_space, g_action_space, l_action_space, pretrain_path, output_size=512, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__() self.net = ObjectNavSLAMNet( observation_space=observation_space, g_action_space=g_action_space, output_size=output_size, obs_transform=obs_transform, pretrain_path=pretrain_path, ) self.num_local_actions = g_action_space.shape[0] self.num_global_actions = g_action_space.shape[0] # print("num_global_actions: %d" % self.num_global_actions) #2 self.global_action_distribution = DiagGaussian(self.net.output_size, self.num_global_actions) self.critic = CriticHead(self.net.output_size)
def __init__( self, observation_space, baseplanes=32, ngroups=32, spatial_size=128, make_backbone=None, use_if_available=["rgb", "depth"], normalize_visual_inputs=True, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__() self.obs_transform = obs_transform if self.obs_transform is not None: observation_space = self.obs_transform.transform_observation_space( observation_space) self._inputs = list( filter(lambda x: x in observation_space.spaces, use_if_available)) self._input_sizes = [0] * len(self._inputs) for i, mode in enumerate(self._inputs): self._input_sizes[i] = observation_space.spaces[mode].shape[2] self.running_mean_and_var = nn.Sequential() if not self.is_blind: spatial_size = observation_space.spaces[ self._inputs[0]].shape[0] // 2 input_channels = sum( self._input_sizes) # self._n_input_depth + self._n_input_rgb self.backbone = make_backbone(input_channels, baseplanes, ngroups) final_spatial = int(spatial_size * self.backbone.final_spatial_compress) after_compression_flat_size = 2048 num_compression_channels = int( round(after_compression_flat_size / (final_spatial**2))) self.compression = nn.Sequential( nn.Conv2d( self.backbone.final_channels, num_compression_channels, kernel_size=3, padding=1, bias=False, ), nn.GroupNorm(1, num_compression_channels), nn.ReLU(True), ) self.output_shape = ( num_compression_channels, final_spatial, final_spatial, )
def __init__( self, observation_space, action_space, goal_sensor_uuid, hidden_size, num_recurrent_layers, rnn_type, backbone, resnet_baseplanes, normalize_visual_inputs, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__() self.goal_sensor_uuid = goal_sensor_uuid self.prev_action_embedding = nn.Embedding(action_space.n + 1, 32) self._n_prev_action = 32 self._n_input_goal = ( observation_space.spaces[self.goal_sensor_uuid].shape[0] + 1) self.tgt_embeding = nn.Linear(self._n_input_goal, 32) self._n_input_goal = 32 self._hidden_size = hidden_size rnn_input_size = self._n_input_goal + self._n_prev_action self.visual_encoder = ResNetEncoder( observation_space, baseplanes=resnet_baseplanes, ngroups=resnet_baseplanes // 2, make_backbone=getattr(resnet, backbone), normalize_visual_inputs=normalize_visual_inputs, obs_transform=obs_transform, ) if not self.visual_encoder.is_blind: self.visual_fc = nn.Sequential( Flatten(), nn.Linear(np.prod(self.visual_encoder.output_shape), hidden_size), nn.ReLU(True), ) self.state_encoder = RNNStateEncoder( (0 if self.is_blind else self._hidden_size) + rnn_input_size, self._hidden_size, rnn_type=rnn_type, num_layers=num_recurrent_layers, ) self.train()
def __init__( self, input_channels, out_channels, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super(MapEncoder, self).__init__() self.maxpool = nn.MaxPool2d(2) self.conv1 = nn.Conv2d(input_channels, 32, 3, stride=1, padding=1) self.conv2 = nn.Conv2d(32, 64, 3, stride=1, padding=1) self.conv3 = nn.Conv2d(64, 128, 3, stride=1, padding=1) self.conv4 = nn.Conv2d(128, 64, 3, stride=1, padding=1) self.conv5 = nn.Conv2d(64, 32, 3, stride=1, padding=1) self.fc = nn.Linear(7200, out_channels)
def __init__( self, observation_space, g_action_space, output_size, pretrain_path, obs_transform=ResizeCenterCropper(size=(240, 240)), ): super().__init__() self._output_size = output_size if ObjectGoalSensor.cls_uuid in observation_space.spaces: self._n_object_categories = (int( observation_space.spaces[ObjectGoalSensor.cls_uuid].high[0]) + 2) self.obj_categories_embedding = nn.Embedding( self._n_object_categories, 256) hidden_size = 256 # current pose embedding curr_pose_dim = observation_space.spaces["curr_pose"].shape[0] # print("curr_pose_dim: ", curr_pose_dim) self.curr_pose_embedding = nn.Linear(curr_pose_dim, 256) hidden_size += 256 map_dim = observation_space.spaces["map_sum"].shape[2] self.map_encoder = MapEncoder( map_dim, 2048, obs_transform=obs_transform, ) hidden_size += 2048 self.linear1 = nn.Linear(hidden_size, 1024) self.linear2 = nn.Linear(1024, self._output_size)
def __init__( self, observation_space, baseplanes=32, ngroups=32, spatial_size=128, make_backbone=None, normalize_visual_inputs=False, obs_transform=ResizeCenterCropper(size=(256, 256)), ): super().__init__() self.obs_transform = obs_transform if self.obs_transform is not None: observation_space = self.obs_transform.transform_observation_space( observation_space) if "rgb" in observation_space.spaces: self._n_input_rgb = observation_space.spaces["rgb"].shape[2] spatial_size = observation_space.spaces["rgb"].shape[0] // 2 else: self._n_input_rgb = 0 if "depth" in observation_space.spaces: self._n_input_depth = observation_space.spaces["depth"].shape[2] spatial_size = observation_space.spaces["depth"].shape[0] // 2 else: self._n_input_depth = 0 if normalize_visual_inputs: self.running_mean_and_var = RunningMeanAndVar(self._n_input_depth + self._n_input_rgb) else: self.running_mean_and_var = nn.Sequential() if not self.is_blind: input_channels = self._n_input_depth + self._n_input_rgb self.backbone = make_backbone(input_channels, baseplanes, ngroups) final_spatial = int(spatial_size * self.backbone.final_spatial_compress) after_compression_flat_size = 2048 num_compression_channels = int( round(after_compression_flat_size / (final_spatial**2))) self.compression = nn.Sequential( nn.Conv2d( self.backbone.final_channels, num_compression_channels, kernel_size=3, padding=1, bias=False, ), nn.GroupNorm(1, num_compression_channels), nn.ReLU(True), ) self.output_shape = ( num_compression_channels, final_spatial, final_spatial, )
def __init__( self, observation_space, action_space, hidden_size, num_recurrent_layers, rnn_type, backbone, resnet_baseplanes, normalize_visual_inputs, obs_transform=ResizeCenterCropper(size=(256, 256)), force_blind_policy=False, ): super().__init__() self.prev_action_embedding = nn.Embedding(action_space.n + 1, 32) self._n_prev_action = 32 rnn_input_size = self._n_prev_action if (IntegratedPointGoalGPSAndCompassSensor.cls_uuid in observation_space.spaces): n_input_goal = (observation_space.spaces[ IntegratedPointGoalGPSAndCompassSensor.cls_uuid].shape[0] + 1) self.tgt_embeding = nn.Linear(n_input_goal, 32) rnn_input_size += 32 if ObjectGoalSensor.cls_uuid in observation_space.spaces: self._n_object_categories = (int( observation_space.spaces[ObjectGoalSensor.cls_uuid].high[0]) + 1) self.obj_categories_embedding = nn.Embedding( self._n_object_categories, 32) rnn_input_size += 32 if EpisodicGPSSensor.cls_uuid in observation_space.spaces: input_gps_dim = observation_space.spaces[ EpisodicGPSSensor.cls_uuid].shape[0] self.gps_embedding = nn.Linear(input_gps_dim, 32) rnn_input_size += 32 if PointGoalSensor.cls_uuid in observation_space.spaces: input_pointgoal_dim = observation_space.spaces[ PointGoalSensor.cls_uuid].shape[0] self.pointgoal_embedding = nn.Linear(input_pointgoal_dim, 32) rnn_input_size += 32 if HeadingSensor.cls_uuid in observation_space.spaces: input_heading_dim = ( observation_space.spaces[HeadingSensor.cls_uuid].shape[0] + 1) assert input_heading_dim == 2, "Expected heading with 2D rotation." self.heading_embedding = nn.Linear(input_heading_dim, 32) rnn_input_size += 32 if ProximitySensor.cls_uuid in observation_space.spaces: input_proximity_dim = observation_space.spaces[ ProximitySensor.cls_uuid].shape[0] self.proximity_embedding = nn.Linear(input_proximity_dim, 32) rnn_input_size += 32 if EpisodicCompassSensor.cls_uuid in observation_space.spaces: assert (observation_space.spaces[EpisodicCompassSensor.cls_uuid]. shape[0] == 1), "Expected compass with 2D rotation." input_compass_dim = 2 # cos and sin of the angle self.compass_embedding = nn.Linear(input_compass_dim, 32) rnn_input_size += 32 if ImageGoalSensor.cls_uuid in observation_space.spaces: goal_observation_space = spaces.Dict( {"rgb": observation_space.spaces[ImageGoalSensor.cls_uuid]}) self.goal_visual_encoder = ResNetEncoder( goal_observation_space, baseplanes=resnet_baseplanes, ngroups=resnet_baseplanes // 2, make_backbone=getattr(resnet, backbone), normalize_visual_inputs=normalize_visual_inputs, obs_transform=obs_transform, ) self.goal_visual_fc = nn.Sequential( Flatten(), nn.Linear(np.prod(self.goal_visual_encoder.output_shape), hidden_size), nn.ReLU(True), ) rnn_input_size += hidden_size self._hidden_size = hidden_size self.visual_encoder = ResNetEncoder( observation_space if not force_blind_policy else spaces.Dict({}), baseplanes=resnet_baseplanes, ngroups=resnet_baseplanes // 2, make_backbone=getattr(resnet, backbone), normalize_visual_inputs=normalize_visual_inputs, obs_transform=obs_transform, ) if not self.visual_encoder.is_blind: self.visual_fc = nn.Sequential( Flatten(), nn.Linear(np.prod(self.visual_encoder.output_shape), hidden_size), nn.ReLU(True), ) self.state_encoder = RNNStateEncoder( (0 if self.is_blind else self._hidden_size) + rnn_input_size, self._hidden_size, rnn_type=rnn_type, num_layers=num_recurrent_layers, ) self.train()
def __init__( self, observation_space, output_size, obs_transform: nn.Module = ResizeCenterCropper(size=(256, 256)), ): super().__init__() self.obs_transform = obs_transform if self.obs_transform is not None: observation_space = obs_transform.transform_observation_space( observation_space) if "rgb" in observation_space.spaces: self._n_input_rgb = observation_space.spaces["rgb"].shape[2] else: self._n_input_rgb = 0 if "depth" in observation_space.spaces: self._n_input_depth = observation_space.spaces["depth"].shape[2] else: self._n_input_depth = 0 # kernel size for different CNN layers self._cnn_layers_kernel_size = [(8, 8), (4, 4), (3, 3)] # strides for different CNN layers self._cnn_layers_stride = [(4, 4), (2, 2), (1, 1)] if self._n_input_rgb > 0: cnn_dims = np.array(observation_space.spaces["rgb"].shape[:2], dtype=np.float32) elif self._n_input_depth > 0: cnn_dims = np.array(observation_space.spaces["depth"].shape[:2], dtype=np.float32) if self.is_blind: self.cnn = nn.Sequential() else: for kernel_size, stride in zip(self._cnn_layers_kernel_size, self._cnn_layers_stride): cnn_dims = self._conv_output_dim( dimension=cnn_dims, padding=np.array([0, 0], dtype=np.float32), dilation=np.array([1, 1], dtype=np.float32), kernel_size=np.array(kernel_size, dtype=np.float32), stride=np.array(stride, dtype=np.float32), ) self.cnn = nn.Sequential( nn.Conv2d( in_channels=self._n_input_rgb + self._n_input_depth, out_channels=32, kernel_size=self._cnn_layers_kernel_size[0], stride=self._cnn_layers_stride[0], ), nn.ReLU(True), nn.Conv2d( in_channels=32, out_channels=64, kernel_size=self._cnn_layers_kernel_size[1], stride=self._cnn_layers_stride[1], ), nn.ReLU(True), nn.Conv2d( in_channels=64, out_channels=32, kernel_size=self._cnn_layers_kernel_size[2], stride=self._cnn_layers_stride[2], ), # nn.ReLU(True), Flatten(), nn.Linear(32 * cnn_dims[0] * cnn_dims[1], output_size), nn.ReLU(True), ) self.layer_init()
def __init__( self, observation_space, g_action_space, output_size, pretrain_path, obs_transform=ResizeCenterCropper(size=(240, 240)), ): super().__init__() self._output_size = output_size # goal encoder if ObjectGoalSensor.cls_uuid in observation_space.spaces: # self.fasttext = torchtext.vocab.FastText() # self.obj_categories_encoder = nn.Linear( # 300, 512 # ) # hidden_size = 512 self._n_object_categories = (int( observation_space.spaces[ObjectGoalSensor.cls_uuid].high[0]) + 2) self.obj_categories_embedding = nn.Embedding( self._n_object_categories, 256) hidden_size = 256 # print('obj_categories_embedding: ', self.obj_categories_embedding) # 22 curr_pose_dim = observation_space.spaces["curr_pose"].shape[0] # print("curr_pose_dim: ", curr_pose_dim) self.curr_pose_embedding = nn.Linear(curr_pose_dim, 256) hidden_size += 256 # map encoder map_dim = observation_space.spaces["map_sum"].shape[2] self.map_encoder = MapEncoder( map_dim, 2048, obs_transform=obs_transform, ) hidden_size += 2048 # scene priors self.semantic_encoder = SemanticMap_Encoder(1, 256) self.graphcnn = GraphRCNN(512, 21) self.edge = torch.tensor( [[ 0, 8, 0, 19, 0, 11, 0, 6, 0, 1, 0, 5, 0, 16, 0, 3, 0, 10, 0, 15, 0, 7, 0, 9, 0, 20, 0, 12, 5, 20, 6, 20, 8, 20, 4, 20, 16, 20, 7, 20, 15, 20, 18, 20, 8, 14, 7, 14, 1, 14, 2, 14, 9, 14, 6, 14, 3, 14, 14, 17, 5, 14, 13, 14, 10, 14, 14, 15, 4, 14, 7, 8, 4, 7, 2, 7, 7, 10, 7, 9, 1, 7, 6, 7, 7, 16, 7, 15, 7, 17, 5, 7, 7, 13, 7, 18, 3, 8, 1, 3, 3, 6, 2, 3, 3, 10, 3, 15, 3, 4, 4, 8, 0, 4, 4, 6, 4, 11, 4, 18, 4, 16, 4, 12, 4, 5, 2, 8, 2, 9, 2, 17, 1, 2, 2, 10, 5, 8, 8, 11, 8, 15, 8, 19, 6, 8, 1, 8, 8, 16, 8, 9, 8, 12, 8, 13, 8, 18, 8, 10, 2, 15, 15, 16, 1, 15, 9, 15, 15, 17, 12, 15, 13, 15, 1, 5, 1, 16, 1, 11, 1, 10, 1, 9, 16, 19, 18, 19, 3, 19, 6, 11, 6, 18, 5, 6, 6, 16, 13, 17, 10, 17, 12, 17, 8, 17, 10, 11, 5, 11, 10, 13, 12, 13, 6, 12, 11, 12, 12, 16, 16, 18, 0, 18 ], [ 8, 0, 19, 0, 11, 0, 6, 0, 1, 0, 5, 0, 16, 0, 3, 0, 10, 0, 15, 0, 7, 0, 9, 0, 20, 0, 12, 0, 20, 5, 20, 6, 20, 8, 20, 4, 20, 16, 20, 7, 20, 15, 20, 18, 14, 8, 14, 7, 14, 1, 14, 2, 14, 9, 14, 6, 14, 3, 17, 14, 14, 5, 14, 13, 14, 10, 15, 14, 14, 4, 8, 7, 7, 4, 7, 2, 10, 7, 9, 7, 7, 1, 7, 6, 16, 7, 15, 7, 17, 7, 7, 5, 13, 7, 18, 7, 8, 3, 3, 1, 6, 3, 3, 2, 10, 3, 15, 3, 4, 3, 8, 4, 4, 0, 6, 4, 11, 4, 18, 4, 16, 4, 12, 4, 5, 4, 8, 2, 9, 2, 17, 2, 2, 1, 10, 2, 8, 5, 11, 8, 15, 8, 19, 8, 8, 6, 8, 1, 16, 8, 9, 8, 12, 8, 13, 8, 18, 8, 10, 8, 15, 2, 16, 15, 15, 1, 15, 9, 17, 15, 15, 12, 15, 13, 5, 1, 16, 1, 11, 1, 10, 1, 9, 1, 19, 16, 19, 18, 19, 3, 11, 6, 18, 6, 6, 5, 16, 6, 17, 13, 17, 10, 17, 12, 17, 8, 11, 10, 11, 5, 13, 10, 13, 12, 12, 6, 12, 11, 16, 12, 18, 16, 18, 0 ]], dtype=torch.long) self.edge_type = torch.tensor([ 2, 2, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 2, 2, 2, 2, 0, 0, 2, 2, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 2, 2, 2, 2, 1, 1, 2, 2, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 2, 2, 2, 2, 1, 1, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 1, 1, 1, 1, 2, 2, 0, 0, 2, 2, 0, 0, 2, 2, 0, 0, 2, 2, 2, 2, 0, 0, 2, 2, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 1, 1, 2, 2, 2, 2, 1, 1, 0, 0, 0, 0, 1, 1, 2, 2, 2, 2, 0, 0, 2, 2, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 ], dtype=torch.long) self.graph_fc = nn.Linear(21 * 21, 256) hidden_size += 256 self.linear1 = nn.Linear(hidden_size, 1024) self.linear2 = nn.Linear(1024, self._output_size)