def forward(self, ob_no):
if (isinstance(ob_no, np.ndarray)):
ob_no = ptu.from_numpy(ob_no)
# TODO: Get the prediction error for ob_no
# HINT: Remember to detach the output of self.f!
f_out = self.f(ob_no).detach() # We don't want f to update
f_hat_out = self.f_hat(ob_no)
error = torch.norm(f_out - f_hat_out, dim=1)
return error
# Find a model from detectron2's model zoo. You can use the https://dl.fbaipublicfiles... url as well
cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url(
"COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x.yaml")
model = build_model(cfg)
checkpointer = DetectionCheckpointer(model)
checkpointer.load(cfg.MODEL.WEIGHTS)
model.eval()
with torch.no_grad():
for i in range(max_num):
group = hfile['virtual/' + str(i)]
obs_batch = group['observations'][()]
obs_batch = ptu.from_numpy(obs_batch.transpose([0, 3, 1, 2]))
import ipdb
ipdb.set_trace()
imgs = [dict(image=obs_batch[i]) for i in range(obs_batch.shape[0])]
outputs = model(imgs)
classes = [
out['instances'].pred_classes.cpu().numpy() for out in outputs
]
boxes = [
out['instances'].pred_boxes.tensor.cpu().numpy() for out in outputs
]
areas = [
out['instances'].pred_boxes.area().cpu().numpy() for out in outputs
def __init__(
self,
input_width,
input_height,
input_channels,
output_size,
kernel_sizes,
n_channels,
strides,
paddings,
hidden_sizes=None,
added_fc_input_size=0,
batch_norm_conv=False,
batch_norm_fc=False,
init_w=1e-4,
hidden_init=nn.init.xavier_uniform_,
hidden_activation=nn.ReLU(),
output_activation=identity,
):
if hidden_sizes is None:
hidden_sizes = []
assert len(kernel_sizes) == \
len(n_channels) == \
len(strides) == \
len(paddings)
super().__init__()
self.hidden_sizes = hidden_sizes
self.input_width = input_width
self.input_height = input_height
self.input_channels = input_channels
self.output_size = output_size
self.output_activation = output_activation
self.hidden_activation = hidden_activation
self.batch_norm_conv = batch_norm_conv
self.batch_norm_fc = batch_norm_fc
self.added_fc_input_size = added_fc_input_size
self.conv_input_length = self.input_width * self.input_height * self.input_channels
self.conv_layers = nn.ModuleList()
self.conv_norm_layers = nn.ModuleList()
self.fc_layers = nn.ModuleList()
self.fc_norm_layers = nn.ModuleList()
for out_channels, kernel_size, stride, padding in \
zip(n_channels, kernel_sizes, strides, paddings):
conv = nn.Conv2d(input_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding)
hidden_init(conv.weight)
conv.bias.data.fill_(0)
conv_layer = conv
self.conv_layers.append(conv_layer)
input_channels = out_channels
xcoords = np.expand_dims(np.linspace(-1, 1, self.input_width),
0).repeat(self.input_height, 0)
ycoords = np.repeat(np.linspace(-1, 1, self.input_height),
self.input_width).reshape(
(self.input_height, self.input_width))
self.coords = from_numpy(
np.expand_dims(np.stack([xcoords, ycoords], 0), 0))
def forward_np(self, ob_no):
ob_no = ptu.from_numpy(ob_no)
error = self.forward(ob_no)
return ptu.to_numpy(error)
def __init__(
self,
input_width,
input_height,
input_channels,
output_size,
kernel_sizes,
n_channels,
strides,
paddings,
hidden_sizes,
lstm_size,
lstm_input_size,
added_fc_input_size=0,
batch_norm_conv=False,
batch_norm_fc=False,
init_w=1e-4,
hidden_init=nn.init.xavier_uniform_,
hidden_activation=nn.ReLU(),
lambda_output_activation=identity,
k=None,
):
if hidden_sizes is None:
hidden_sizes = []
assert len(kernel_sizes) == \
len(n_channels) == \
len(strides) == \
len(paddings)
super().__init__()
self.hidden_sizes = hidden_sizes
self.input_width = input_width
self.input_height = input_height
self.input_channels = input_channels
self.lstm_size = lstm_size
self.output_size = output_size
self.lambda_output_activation = lambda_output_activation
self.hidden_activation = hidden_activation
self.batch_norm_conv = batch_norm_conv
self.batch_norm_fc = batch_norm_fc
self.added_fc_input_size = added_fc_input_size
self.conv_input_length = self.input_width * self.input_height * self.input_channels
self.K = k
self.conv_layers = nn.ModuleList()
self.conv_norm_layers = nn.ModuleList()
self.fc_layers = nn.ModuleList()
self.fc_norm_layers = nn.ModuleList()
self.avg_pooling = torch.nn.AvgPool2d(kernel_size=input_width)
self.lstm = nn.LSTM(lstm_input_size,
lstm_size,
num_layers=1,
batch_first=True)
for out_channels, kernel_size, stride, padding in \
zip(n_channels, kernel_sizes, strides, paddings):
conv = nn.Conv2d(input_channels,
out_channels,
kernel_size,
stride=stride,
padding=padding)
hidden_init(conv.weight)
conv.bias.data.fill_(0)
conv_layer = conv
self.conv_layers.append(conv_layer)
input_channels = out_channels
# find output dim of conv_layers by trial and add normalization conv layers
test_mat = torch.zeros(
1, self.input_channels, self.input_width, self.input_height
) # initially the model is on CPU (caller should then move it to GPU if
for conv_layer in self.conv_layers:
test_mat = conv_layer(test_mat)
#self.conv_norm_layers.append(nn.BatchNorm2d(test_mat.shape[1]))
test_mat = self.avg_pooling(test_mat) #Avg pooling layer
fc_input_size = int(np.prod(test_mat.shape))
# used only for injecting input directly into fc layers
fc_input_size += added_fc_input_size
for idx, hidden_size in enumerate(hidden_sizes):
fc_layer = nn.Linear(fc_input_size, hidden_size)
#norm_layer = nn.BatchNorm1d(hidden_size)
fc_layer.weight.data.uniform_(-init_w, init_w)
fc_layer.bias.data.uniform_(-init_w, init_w)
self.fc_layers.append(fc_layer)
#self.fc_norm_layers.append(norm_layer)
fc_input_size = hidden_size
self.last_fc = nn.Linear(lstm_size, output_size)
#self.last_fc.weight.data.uniform_(-init_w, init_w)
#self.last_fc.bias.data.uniform_(-init_w, init_w)
self.last_fc2 = nn.Linear(lstm_size, output_size)
xcoords = np.expand_dims(np.linspace(-1, 1, self.input_width),
0).repeat(self.input_height, 0)
ycoords = np.repeat(np.linspace(-1, 1, self.input_height),
self.input_width).reshape(
(self.input_height, self.input_width))
self.coords = from_numpy(
np.expand_dims(np.stack([xcoords, ycoords], 0), 0)) #(1, 2, D, D)