def __init__(self, uuid: str = "target_coordinates_ind", **kwargs: Any):
observation_space = gym.spaces.Box(
low=np.finfo(np.float32).min,
high=np.finfo(np.float32).max,
shape=(2,),
dtype=np.float32,
)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
mean: Optional[np.ndarray] = None,
stdev: Optional[np.ndarray] = None,
height: Optional[int] = None,
width: Optional[int] = None,
uuid: str = "vision",
output_shape: Optional[Tuple[int, ...]] = None,
output_channels: Optional[int] = None,
unnormalized_infimum: float = -np.inf,
unnormalized_supremum: float = np.inf,
scale_first: bool = True,
**kwargs: Any):
"""Initializer.
# Parameters
config : The images will be normalized
with means `config["mean"]` and standard deviations `config["stdev"]`. If both `config["height"]` and
`config["width"]` are non-negative integers then
the image returned from the environment will be rescaled to have
`config["height"]` rows and `config["width"]` columns using bilinear sampling. The universally unique
identifier will be set as `config["uuid"]`.
args : Extra args. Currently unused.
kwargs : Extra kwargs. Currently unused.
"""
self._norm_means = mean
self._norm_sds = stdev
assert (self._norm_means is None) == (self._norm_sds is None), (
"In VisionSensor's config, "
"either both mean/stdev must be None or neither.")
self._should_normalize = self._norm_means is not None
self._height = height
self._width = width
assert (self._width is None) == (self._height is None), (
"In VisionSensor's config, "
"either both height/width must be None or neither.")
self._scale_first = scale_first
self.scaler: Optional[ScaleBothSides] = None
if self._width is not None:
self.scaler = ScaleBothSides(width=cast(int, self._width),
height=cast(int, self._height))
self.to_pil = transforms.ToPILImage(
) # assumes mode="RGB" for 3 channels
observation_space = self._get_observation_space(
output_shape=output_shape,
output_channels=output_channels,
unnormalized_infimum=unnormalized_infimum,
unnormalized_supremum=unnormalized_supremum,
)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
coordinate_dims: int,
uuid: str = "target_coordinates_ind",
**kwargs: Any):
# Distance is a non-negative real and angle is normalized to the range (-Pi, Pi] or [-Pi, Pi)
observation_space = gym.spaces.Box(np.float32(-3.15),
np.float32(1000),
shape=(coordinate_dims, ))
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
nactions: int,
uuid: str = "expert_policy",
expert_args: Optional[Dict[str, Any]] = None,
**kwargs: Any) -> None:
self.nactions = nactions
self.expert_args: Dict[str, Any] = expert_args or {}
observation_space = self._get_observation_space()
super().__init__(**prepare_locals_for_super(locals()))
def __init__(
self,
input_uuids: List[str],
output_uuid: str,
input_height: int,
input_width: int,
max_dets: int,
detector_spatial_res: int,
detector_thres: float,
device: Optional[torch.device] = None,
device_ids: Optional[List[torch.device]] = None,
**kwargs: Any,
):
self.input_height = input_height
self.input_width = input_width
self.max_dets = max_dets
self.detector_spatial_res = detector_spatial_res
self.detector_thres = detector_thres
self.device = torch.device("cpu") if device is None else device
self.device_ids = device_ids or cast(
List[torch.device], list(range(torch.cuda.device_count())))
self.frcnn: BatchedFasterRCNN = BatchedFasterRCNN(
thres=self.detector_thres,
maxdets=self.max_dets,
res=self.detector_spatial_res,
)
spaces: OrderedDict[str, gym.Space] = OrderedDict()
shape = (self.max_dets, self.detector_spatial_res,
self.detector_spatial_res)
spaces["frcnn_classes"] = gym.spaces.Box(
low=0, # 0 is bg
high=len(self.COCO_INSTANCE_CATEGORY_NAMES) - 1,
shape=shape,
dtype=np.int64,
)
shape = (
self.max_dets * 5,
self.detector_spatial_res,
self.detector_spatial_res,
)
spaces["frcnn_boxes"] = gym.spaces.Box(low=-np.inf,
high=np.inf,
shape=shape)
assert (
len(input_uuids) == 1
), "fasterrcnn preprocessor can only consume one observation type"
observation_space = SpaceDict(spaces=spaces)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(
self,
use_resnet_normalization: bool = True,
mean: Optional[np.ndarray] = np.array([[[0.485, 0.456, 0.406]]],
dtype=np.float32),
stdev: Optional[np.ndarray] = np.array([[[0.229, 0.224, 0.225]]],
dtype=np.float32),
height: Optional[int] = None,
width: Optional[int] = None,
uuid: str = "rgb",
output_shape: Optional[Tuple[int, ...]] = (2048, ),
output_channels: Optional[int] = None,
unnormalized_infimum: float = -np.inf,
unnormalized_supremum: float = np.inf,
scale_first: bool = False,
**kwargs: Any):
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
input_uuids: List[str],
output_uuid: str,
input_height: int,
input_width: int,
output_height: int,
output_width: int,
output_dims: int,
pool: bool,
torchvision_resnet_model: Callable[
..., models.ResNet] = models.resnet18,
device: Optional[torch.device] = None,
device_ids: Optional[List[torch.device]] = None,
**kwargs: Any):
def f(x, k):
assert k in x, "{} must be set in ResNetPreprocessor".format(k)
return x[k]
def optf(x, k, default):
return x[k] if k in x else default
self.input_height = input_height
self.input_width = input_width
self.output_height = output_height
self.output_width = output_width
self.output_dims = output_dims
self.pool = pool
self.make_model = torchvision_resnet_model
self.device = torch.device("cpu") if device is None else device
self.device_ids = device_ids or cast(
List[torch.device], list(range(torch.cuda.device_count())))
self._resnet: Optional[ResNetEmbedder] = None
low = -np.inf
high = np.inf
shape = (self.output_dims, self.output_height, self.output_width)
assert (len(input_uuids) == 1
), "resnet preprocessor can only consume one observation type"
observation_space = gym.spaces.Box(low=low, high=high, shape=shape)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
mean: Optional[np.ndarray] = None,
stdev: Optional[np.ndarray] = None,
height: Optional[int] = None,
width: Optional[int] = None,
uuid: str = "resnet",
output_shape: Optional[Tuple[int, ...]] = None,
output_channels: Optional[int] = None,
unnormalized_infimum: float = -np.inf,
unnormalized_supremum: float = np.inf,
scale_first: bool = True,
**kwargs: Any):
self.to_tensor = transforms.ToTensor()
self.resnet = nn.Sequential(
*list(models.resnet50(pretrained=True).children())[:-1] +
[Flatten()]).eval()
self.device: torch.device = torch.device("cpu")
super().__init__(**prepare_locals_for_super(locals()))
def __init__(
self,
action_space: gym.spaces.Discrete,
observation_space: SpaceDict,
num_objects: int,
num_colors: int,
num_states: int,
object_embedding_dim: int = 8,
hidden_size=512,
num_layers=1,
rnn_type="GRU",
head_type: Callable[
..., ActorCriticModel[CategoricalDistr]] = LinearActorCritic,
**kwargs,
):
super().__init__(**prepare_locals_for_super(locals()))
self._hidden_size = hidden_size
agent_view_x, agent_view_y, view_channels = observation_space[
"minigrid_ego_image"].shape
self.actor_critic = RNNActorCritic(
input_uuid=self.ac_key,
action_space=action_space,
observation_space=SpaceDict({
self.ac_key:
gym.spaces.Box(
low=np.float32(-1.0),
high=np.float32(1.0),
shape=(self.object_embedding_dim * agent_view_x *
agent_view_y * view_channels, ),
)
}),
hidden_size=hidden_size,
num_layers=num_layers,
rnn_type=rnn_type,
head_type=head_type,
)
self.memory_key = "rnn"
self.train()
def __init__(self,
object_types: List[str],
target_to_detector_map: Optional[Dict[str, str]] = None,
detector_types: Optional[List[str]] = None,
uuid: str = "goal_object_type_ind",
**kwargs: Any):
self.ordered_object_types = list(object_types)
assert self.ordered_object_types == sorted(
self.ordered_object_types
), "object types input to goal object type sensor must be ordered"
if target_to_detector_map is None:
self.object_type_to_ind = {
ot: i
for i, ot in enumerate(self.ordered_object_types)
}
observation_space = gym.spaces.Discrete(
len(self.ordered_object_types))
else:
assert (detector_types
is not None), "Missing detector_types for map {}".format(
target_to_detector_map)
self.target_to_detector = target_to_detector_map
self.detector_types = detector_types
detector_index = {
ot: i
for i, ot in enumerate(self.detector_types)
}
self.object_type_to_ind = {
ot: detector_index[self.target_to_detector[ot]]
for ot in self.ordered_object_types
}
observation_space = gym.spaces.Discrete(len(self.detector_types))
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self,
use_resnet_normalization: Optional[bool] = None,
use_normalization: Optional[bool] = None,
mean: Optional[np.ndarray] = np.array([[0.5]],
dtype=np.float32),
stdev: Optional[np.ndarray] = np.array([[0.25]],
dtype=np.float32),
height: Optional[int] = None,
width: Optional[int] = None,
uuid: str = "depth",
output_shape: Optional[Tuple[int, ...]] = None,
output_channels: int = 1,
unnormalized_infimum: float = 0.0,
unnormalized_supremum: float = 5.0,
scale_first: bool = False,
**kwargs: Any):
# Give priority to use_normalization, but use_resnet_normalization for backward compat. if not set
if use_resnet_normalization is not None and use_normalization is None:
use_normalization = use_resnet_normalization
elif use_normalization is None:
use_normalization = False
super().__init__(**prepare_locals_for_super(locals()))
def __init__(self, uuid: str = "target_object_id", **kwargs: Any):
observation_space = gym.spaces.Discrete(38)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(
self,
input_uuids: List[str],
output_uuid: str,
input_height: int,
input_width: int,
output_height: int,
output_width: int,
output_dims: int,
pool: bool,
torchvision_resnet_model: Callable[..., models.ResNet] = models.resnet18,
parallel: bool = True,
device: Optional[torch.device] = None,
device_ids: Optional[List[torch.device]] = None,
**kwargs: Any
):
def f(x, k):
assert k in x, "{} must be set in ResnetPreProcessorHabitat".format(k)
return x[k]
def optf(x, k, default):
return x[k] if k in x else default
self.input_height = input_height
self.input_width = input_width
self.output_height = output_height
self.output_width = output_width
self.output_dims = output_dims
self.pool = pool
self.make_model = torchvision_resnet_model
self.parallel = parallel
self.device = (
device
if device is not None
else ("cuda" if self.parallel and torch.cuda.is_available() else "cpu")
)
self.device_ids = device_ids or cast(
List[torch.device], list(range(torch.cuda.device_count()))
)
self.resnet: Union[
ResNetEmbedder, torch.nn.DataParallel[ResNetEmbedder]
] = ResNetEmbedder(
self.make_model(pretrained=True).to(self.device), pool=self.pool
)
if self.parallel:
assert (
torch.cuda.is_available()
), "attempt to parallelize resnet without cuda"
get_logger().info("Distributing resnet")
self.resnet = self.resnet.to(torch.device("cuda"))
# store = torch.distributed.TCPStore("localhost", 4712, 1, True)
# torch.distributed.init_process_group(backend="nccl", store=store, rank=0, world_size=1)
# self.model = DistributedDataParallel(self.frcnn, device_ids=self.device_ids)
self.resnet = torch.nn.DataParallel(
self.resnet, device_ids=self.device_ids
) # , output_device=torch.cuda.device_count() - 1)
get_logger().info("Detected {} devices".format(torch.cuda.device_count()))
low = -np.inf
high = np.inf
shape = (self.output_dims, self.output_height, self.output_width)
assert (
len(input_uuids) == 1
), "resnet preprocessor can only consume one observation type"
observation_space = gym.spaces.Box(low=low, high=high, shape=shape)
super().__init__(**prepare_locals_for_super(locals()))
def __init__(
self,
input_uuids: List[str],
output_uuid: str,
input_height: int,
input_width: int,
max_dets: int,
detector_spatial_res: int,
detector_thres: float,
parallel: bool = False,
device: Optional[torch.device] = None,
device_ids: Optional[List[torch.device]] = None,
**kwargs: Any,
):
self.input_height = input_height
self.input_width = input_width
self.max_dets = max_dets
self.detector_spatial_res = detector_spatial_res
self.detector_thres = detector_thres
self.parallel = parallel
self.device = (device if device is not None else (
"cuda" if self.parallel and torch.cuda.is_available() else "cpu"))
self.device_ids = device_ids or cast(
List[torch.device], list(range(torch.cuda.device_count())))
self.frcnn: Union[BatchedFasterRCNN,
torch.nn.DataParallel] = BatchedFasterRCNN(
thres=self.detector_thres,
maxdets=self.max_dets,
res=self.detector_spatial_res,
)
if self.parallel:
assert (torch.cuda.is_available()
), "attempt to parallelize detector without cuda"
get_logger().info("Distributing detector")
self.frcnn = self.frcnn.to(torch.device("cuda"))
self.frcnn = torch.nn.DataParallel(self.frcnn,
device_ids=self.device_ids)
get_logger().info("Detected {} devices".format(
torch.cuda.device_count()))
spaces: OrderedDict[str, gym.Space] = OrderedDict()
shape = (self.max_dets, self.detector_spatial_res,
self.detector_spatial_res)
spaces["frcnn_classes"] = gym.spaces.Box(
low=0, # 0 is bg
high=len(self.COCO_INSTANCE_CATEGORY_NAMES) - 1,
shape=shape,
dtype=np.int64,
)
shape = (
self.max_dets * 5,
self.detector_spatial_res,
self.detector_spatial_res,
)
spaces["frcnn_boxes"] = gym.spaces.Box(low=-np.inf,
high=np.inf,
shape=shape)
assert (
len(input_uuids) == 1
), "fasterrcnn preprocessor can only consume one observation type"
observation_space = SpaceDict(spaces=spaces)
super().__init__(**prepare_locals_for_super(locals()))
