def get_paf_and_heatmap(model, img_raw, scale_search, param_stride=8, box_size=368):
multiplier = [scale * box_size / img_raw.shape[0] for scale in scale_search]
heatmap_avg = torch.zeros((len(multiplier), 19, img_raw.shape[0], img_raw.shape[1])).cuda()
paf_avg = torch.zeros((len(multiplier), 38, img_raw.shape[0], img_raw.shape[1])).cuda()
for i, scale in enumerate(multiplier):
img_test = cv2.resize(img_raw, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_CUBIC)
img_test_pad, pad = pad_right_down_corner(img_test, param_stride, param_stride)
img_test_pad = np.transpose(np.float32(img_test_pad[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5
feed = Variable(torch.from_numpy(img_test_pad)).cuda()
output1, output2 = model(feed)
print(output1.size())
print(output2.size())
heatmap = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output2)
paf = nn.UpsamplingBilinear2d((img_raw.shape[0], img_raw.shape[1])).cuda()(output1)
heatmap_avg[i] = heatmap[0].data
paf_avg[i] = paf[0].data
heatmap_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(heatmap_avg, 0)), 0, 1), 1, 2).cuda()
heatmap_avg = heatmap_avg.cpu().numpy()
paf_avg = torch.transpose(torch.transpose(torch.squeeze(torch.mean(paf_avg, 0)), 0, 1), 1, 2).cuda()
paf_avg = paf_avg.cpu().numpy()
return paf_avg, heatmap_avg
def forward(self, X, X_mask, verbose=False):
#X: [m, Tx] m = batch size, Tx = word count
if verbose: print(X.size(), type(X))
m = X.size()[0]
Tx = X.size()[1]
#embedding layer
X = self.embedding(X)
#X: [m, Tx, embedding_dim]
if verbose: print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, self.embedding_dim])
#LSTM1
# Transpose batch and sequence dims
X = X.transpose(0, 1)
X, _ = self.LSTM1(X)
# Transpose back
X = X.transpose(0, 1)
#X: [m, Tx, LSTM1_hidden_size]
if verbose: print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, self.LSTM1_hidden_size])
#assert X.size() == torch.Size([m, Tx, 2*self.LSTM1_hidden_size])
#dropout
X = self.dropout(X)
#LSTM2, reduce dimension
# Transpose batch and sequence dims
X = X.transpose(0, 1)
_, X = self.LSTM2(X)
X = X[0]
# Transpose back
X = X.transpose(0, 1)
X = torch.squeeze(X)
#X: [m, LSTM2_hidden_size]
if verbose: print(X.size(), type(X.data))
assert X.size() == torch.Size([m, self.LSTM2_hidden_size])
#dropout
X = self.dropout(X)
#linear
X = self.linear(X)
#X: [m, 1]
if verbose: print(X.size(), type(X))
if self.num_classes == 2:
assert X.size() == torch.Size([m, 1])
else:
assert X.size() == torch.Size([m, self.num_classes])
if self.num_classes == 2:
X = torch.squeeze(X)
X = self.sigmoid(X)
#X: [m]
if verbose: print(X.size(), type(X))
assert X.size() == torch.Size([m])
return X
else:
return F.softmax(X)
def plot_isocontours_expected(ax, model, data, xlimits=[-6, 6], ylimits=[-6, 6],
numticks=101, cmap=None, alpha=1., legend=False):
x = np.linspace(*xlimits, num=numticks)
y = np.linspace(*ylimits, num=numticks)
X, Y = np.meshgrid(x, y)
# zs = np.exp(func(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T))
aaa = torch.from_numpy(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T).type(torch.FloatTensor)
n_samps = 10
if len(data) < n_samps:
n_samps = len(data)
for samp_i in range(n_samps):
if samp_i % 1000 == 0:
print samp_i
mean, logvar = model.encode(Variable(torch.unsqueeze(data[samp_i],0)))
func = lambda zs: lognormal4(torch.Tensor(zs), torch.squeeze(mean.data), torch.squeeze(logvar.data))
# print aaa.size()
bbb = func(aaa)
# print 'sum:1', torch.sum(bbb)
ddd = torch.exp(bbb)
# print 'sum:', torch.sum(ddd)
# print ddd.size()
# fdsa
if samp_i ==0:
sum_of_all = ddd
else:
sum_of_all = sum_of_all + ddd
avg_of_all = sum_of_all / n_samps
Z = avg_of_all.view(X.shape)
Z=Z.numpy()
# print 'sum:', np.sum(Z)
cs = plt.contour(X, Y, Z, cmap=cmap, alpha=alpha)
if legend:
nm, lbl = cs.legend_elements()
plt.legend(nm, lbl, fontsize=4)
ax.set_yticks([])
ax.set_xticks([])
plt.gca().set_aspect('equal', adjustable='box')
return Z
def plot_isocontours_expected_norm_ind(ax, model, data, xlimits=[-6, 6], ylimits=[-6, 6],
numticks=101, cmap=None, alpha=1., legend=False, n_samps=10, cs_to_use=None):
x = np.linspace(*xlimits, num=numticks)
y = np.linspace(*ylimits, num=numticks)
X, Y = np.meshgrid(x, y)
# zs = np.exp(func(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T))
aaa = torch.from_numpy(np.concatenate([np.atleast_2d(X.ravel()), np.atleast_2d(Y.ravel())]).T).type(torch.FloatTensor)
# n_samps = 10
if len(data) < n_samps:
n_samps = len(data)
for samp_i in range(n_samps):
if samp_i % 1000 == 0:
print samp_i
mean, logvar = model.encode(Variable(torch.unsqueeze(data[samp_i],0)))
func = lambda zs: lognormal4(torch.Tensor(zs), torch.squeeze(mean.data), torch.squeeze(logvar.data))
# print aaa.size()
bbb = func(aaa)
zs = bbb.numpy()
max_ = np.max(zs)
zs_sum = np.log(np.sum(np.exp(zs-max_))) + max_
zs = zs - zs_sum
ddd = np.exp(zs)
Z = ddd
Z = Z.reshape(X.shape)
if cs_to_use != None:
cs = plt.contour(X, Y, Z, cmap=cmap, alpha=alpha, levels=cs_to_use.levels)
else:
cs = plt.contour(X, Y, Z, cmap=cmap, alpha=alpha)
# if samp_i ==0:
# sum_of_all = ddd
# else:
# sum_of_all = sum_of_all + ddd
# avg_of_all = sum_of_all / n_samps
# Z = avg_of_all.reshape(X.shape)
# print 'sum:', np.sum(Z)
# if legend:
# nm, lbl = cs.legend_elements()
# plt.legend(nm, lbl, fontsize=4)
ax.set_yticks([])
ax.set_xticks([])
plt.gca().set_aspect('equal', adjustable='box')
return Z, cs
def box_loss(self,gt_label,gt_offset,pred_offset):
#get the mask element which != 0
mask = torch.ne(gt_label,0)
#convert mask to dim index
chose_index = torch.nonzero(mask)
chose_index = torch.squeeze(chose_index)
#only valid element can effect the loss
valid_gt_offset = gt_offset[chose_index,:]
valid_pred_offset = pred_offset[chose_index,:]
valid_pred_offset = torch.squeeze(valid_pred_offset)
return self.loss_box(valid_pred_offset,valid_gt_offset)
def forward(self, x):
out = self.conv1(x)
for i in range(1, self.n_layers):
dense_layer = getattr(self, f'dense{i}')
trans_layer = getattr(self, f'trans{i}')
out = trans_layer(dense_layer(out))
last_dense_layer = getattr(self, f'dense{self.n_layers}')
out = last_dense_layer(out)
out = F.avg_pool2d(F.relu(self.bn1(out)), out.size()[-1])
out = torch.squeeze(torch.squeeze(out, 2), 2)
out = F.log_softmax(self.fc(out))
return out
def cross_entropy2d(input, target, weight=None, size_average=True):
"""
Function to compute pixelwise cross-entropy for 2D image. This is the segmentation loss.
Args:
input: input tensor of shape (minibatch x num_channels x h x w)
target: 2D label map of shape (minibatch x h x w)
weight (optional): tensor of size 'C' specifying the weights to be given to each class
size_average (optional): boolean value indicating whether the NLL loss has to be normalized
by the number of pixels in the image
"""
# input: (n, c, h, w), target: (n, h, w)
n, c, h, w = input.size()
# log_p: (n, c, h, w)
log_p = F.log_softmax(input)
# log_p: (n*h*w, c)
log_p = log_p.transpose(1, 2).transpose(2, 3).contiguous().view(-1, c)
try:
log_p = log_p[target.view(n, h, w, 1).repeat(1, 1, 1, c) >= 0]
except:
print "Exception: ", target.size()
log_p = log_p.view(-1, c)
# target: (n*h*w,)
mask = target >= 0
target = target[mask]
target = torch.squeeze(target)
loss = F.nll_loss(log_p, target, weight=weight, size_average=False)
if size_average:
loss /= mask.data.sum()
return loss
def forward(self, X, X_mask):
#X: [m, Tx] m = batch size, Tx = word count
#print(X.size(), type(X))
m = X.size()[0]
Tx = X.size()[1]
X = self.embedding(X)
#X: [m, Tx, embedding_dim] m = batch size, Tx = word count
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, self.embedding_dim])
#average words in doc. use mask so we average only words not padding
X = torch.sum(X, 1)
X = Variable(torch.div(X.data, X_mask))
#X: [m, emb_dim]
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, self.embedding_dim])
X = self.linear(X)
#X: [m, 1]
#print(X.size(), type(X))
if self.num_classes == 2:
assert X.size() == torch.Size([m, 1])
else:
assert X.size() == torch.Size([m, self.num_classes])
if self.num_classes == 2:
X = torch.squeeze(X)
X = self.sigmoid(X)
#X: [m]
#print(X.size(), type(X))
assert X.size() == torch.Size([m])
return X
else:
return F.softmax(X)
def __bool__(self):
if self.numel() == 0:
return False
elif self.numel() == 1:
return torch.squeeze(self)[0] != 0
raise RuntimeError("bool value of " + torch.typename(self) +
" containing more than one value is ambiguous")
def test(net, testloader, config):
total, correct = 0.0, 0.0
for i, data in enumerate(testloader):
# Get inputs
X, S1, S2, labels = data
if X.size()[0] != config.batch_size:
continue # Drop those data, if not enough for a batch
# Send Tensors to GPU if available
if use_GPU:
X = X.cuda()
S1 = S1.cuda()
S2 = S2.cuda()
labels = labels.cuda()
# Wrap to autograd.Variable
X, S1, S2 = Variable(X), Variable(S1), Variable(S2)
# Forward pass
outputs, predictions = net(X, S1, S2, config)
# Select actions with max scores(logits)
_, predicted = torch.max(outputs, dim=1, keepdim=True)
# Unwrap autograd.Variable to Tensor
predicted = predicted.data
# Compute test accuracy
correct += (torch.eq(torch.squeeze(predicted), labels)).sum()
total += labels.size()[0]
print('Test Accuracy: {:.2f}%'.format(100 * (correct / total)))
def _get_state_action_values(self, transitions):
batch_size = len(transitions)
batch = Transition(*zip(*transitions))
non_final_mask = torch.tensor(tuple(map(lambda s: s is not None, batch.next_state)), dtype=torch.uint8, device=self.config.device)
state_batch = torch.cat(batch.state).to(torch.float32)
action_batch = torch.cat(batch.action)
reward_batch = torch.cat(batch.reward).to(torch.float32)
next_state_values = torch.zeros(batch_size).to(self.config.device, dtype=torch.float32)
next_states = [s for s in batch.next_state if s is not None]
if len(next_states) != 0:
with torch.no_grad():
non_final_next_state = torch.cat(next_states).to(torch.float32)
Q = self._get_Q(self.model, non_final_next_state)
best_actions = torch.argmax(Q, dim=1, keepdim=True)
Q_target = self._get_Q(self.target_model, non_final_next_state)
next_state_values[non_final_mask] = Q_target.gather(1, best_actions).squeeze()
gamma = self.config.gamma ** self.config.num_multi_step_reward
expected_values = reward_batch + gamma * next_state_values
with torch.set_grad_enabled(self.model.training):
Q = self._get_Q(self.model, state_batch)
values = torch.squeeze(Q.gather(1, action_batch))
values.to(self.config.device, dtype=torch.float32)
return (values, expected_values)
def cls_loss(self,gt_label,pred_label):
# get the mask element which >= 0, only 0 and 1 can effect the detection loss
pred_label = torch.squeeze(pred_label)
mask = torch.ge(gt_label,0)
valid_gt_label = torch.masked_select(gt_label,mask).float()
valid_pred_label = torch.masked_select(pred_label,mask)
return self.loss_cls(valid_pred_label,valid_gt_label)
def forward(self, x):
x = self.model.conv1(x)
x = self.model.bn1(x)
x = self.model.relu(x)
x = self.model.maxpool(x)
x = self.model.layer1(x)
x = self.model.layer2(x)
x = self.model.layer3(x)
x = self.model.layer4(x)
x = self.avgpool(x)
x = self.dropout(x)
part = {}
predict = {}
# get six part feature batchsize*2048*6
for i in range(self.part):
part[i] = torch.squeeze(x[:,:,i])
name = 'classifier'+str(i)
c = getattr(self,name)
predict[i] = c(part[i])
# sum prediction
#y = predict[0]
#for i in range(self.part-1):
# y += predict[i+1]
y = []
for i in range(self.part):
y.append(predict[i])
return y
def reply(self):
if (len(self.memory) < BATCH_SIZE):
return
transitions = self.memory.sample(BATCH_SIZE)
batch = Transition(*zip(*transitions))
non_final_mask = torch.ByteTensor(tuple(map(lambda s: s is not None, batch.next_state)))
state_batch = torch.cat(batch.state)
action_batch = torch.cat(batch.action)
reward_batch = torch.cat(batch.reward)
non_final_next_state = torch.cat([s for s in batch.next_state if s is not None])
self.model.eval()
state_action_values = torch.squeeze(self.model(state_batch).gather(1, action_batch))
next_state_values = torch.zeros(BATCH_SIZE).type(torch.FloatTensor)
next_state_values[non_final_mask] = self.model(non_final_next_state).data.max(1)[0]
expected_state_action_values = reward_batch + GAMMA * next_state_values
self.model.train()
loss = F.smooth_l1_loss(state_action_values, expected_state_action_values)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
def extract_features(im, test_model=None, layers=('layer3',)):
"""
Args:
im (np.ndarray): Image, read using cv2.imread so is in BGR format.
Returns:
features (list): List of features from each layer in the list layers.
"""
model = test_model or default_model
model.eval()
# Preprocess the image
im = prepare_image(im)
# Extract the features
x = im
outputs = []
layers = list(layers)
for name, module in model._modules.items():
if len(layers) == 0:
break
if name == 'fc':
# Not sure why I need to do this...
x = torch.squeeze(x)
x = module.cuda()(x)
if name in layers:
outputs += [x.data.cpu().clone().numpy()]
del layers[layers.index(name)]
return outputs
def forward(self, x):
out = self.conv1(x)
out = self.trans1(self.dense1(out))
out = self.trans2(self.dense2(out))
out = self.dense3(out)
out = torch.squeeze(F.avg_pool2d(F.relu(self.bn1(out)), 8))
out = F.log_softmax(self.fc(out))
return out
def image_classification_test(loader, model, test_10crop=True, gpu=True):
start_test = True
if test_10crop:
iter_test = [iter(loader['test' + str(i)]) for i in range(10)]
for i in range(len(loader['test0'])):
data = [iter_test[j].next() for j in range(10)]
inputs = [data[j][0] for j in range(10)]
labels = data[0][1]
for j in range(10):
inputs[j] = torch.Tensor(inputs[j], device=device)
labels = torch.Tensor(labels, device=device)
outputs = []
for j in range(10):
outputs.append(model(inputs[j]))
outputs = sum(outputs)
if start_test:
all_output = outputs.data.float()
all_label = labels.data.float()
start_test = False
else:
all_output = torch.cat((all_output, outputs.data.float()), 0)
all_label = torch.cat((all_label, labels.data.float()), 0)
else:
iter_test = iter(loader["test"])
data = iter_test.next()
inputs = data[0]
labels = data[1]
inputs = inputs.to(device)
labels = labels.to(device)
outputs = model(inputs)
if start_test:
all_output = outputs.data.float()
all_label = labels.data.float()
start_test = False
else:
all_output = torch.cat((all_output, outputs.data.float()), 0)
all_label = torch.cat((all_label, labels.data.float()), 0)
_, predict = torch.max(all_output, 1)
torch.sum(torch.squeeze(predict).float() == all_label)
accuracy = float(torch.sum(torch.squeeze(predict).float() == all_label)) / float(all_label.size()[0])
return accuracy
def testModulus(self):
for jit in [True, False]:
modulus = sl.Modulus(jit=jit)
x = torch.cuda.FloatTensor(100,10,4,2).copy_(torch.rand(100,10,4,2))
y = modulus(x)
u = torch.squeeze(torch.sqrt(torch.sum(x * x, 3)))
v = y.narrow(3, 0, 1)
self.assertAlmostEqual(linfnorm(u, v), 0, places=6)
def forward(self, x):
if len(x.size()) == 3: # N x k xdim
# N x dim x k x 1
x_reshape = torch.unsqueeze(x.permute(0, 2, 1), 3)
elif len(x.size()) == 2: # N x dim
# N x dim x 1 x 1
x_reshape = torch.unsqueeze(torch.unsqueeze(x, 2), 3)
iatt_conv1 = self.conv1(x_reshape) # N x hidden_dim x * x 1
iatt_relu = F.relu(iatt_conv1)
iatt_conv2 = self.conv2(iatt_relu) # N x out_dim x * x 1
if len(x.size()) == 3:
iatt_conv3 = torch.squeeze(iatt_conv2, 3).permute(0, 2, 1)
elif len(x.size()) == 2:
iatt_conv3 = torch.squeeze(torch.squeeze(iatt_conv2, 3), 2)
return iatt_conv3
def landmark_loss(self,gt_label,gt_landmark,pred_landmark):
mask = torch.eq(gt_label,-2)
chose_index = torch.nonzero(mask.data)
chose_index = torch.squeeze(chose_index)
valid_gt_landmark = gt_landmark[chose_index, :]
valid_pred_landmark = pred_landmark[chose_index, :]
return self.loss_landmark(valid_pred_landmark, valid_gt_landmark)
def visualize_image(self, writer, dataset, image, target, output, global_step):
grid_image = make_grid(image[:3].clone().cpu().data, 3, normalize=True)
writer.add_image('Image', grid_image, global_step)
grid_image = make_grid(decode_seg_map_sequence(torch.max(output[:3], 1)[1].detach().cpu().numpy(),
dataset=dataset), 3, normalize=False, range=(0, 255))
writer.add_image('Predicted label', grid_image, global_step)
grid_image = make_grid(decode_seg_map_sequence(torch.squeeze(target[:3], 1).detach().cpu().numpy(),
dataset=dataset), 3, normalize=False, range=(0, 255))
writer.add_image('Groundtruth label', grid_image, global_step)
def main(args):
dataset = load_config(args.dataset)
path = dataset['common']['dataset']
train_transform = Compose([
ConvertImageMode(mode='RGB'),
ImageToTensor()
])
train_dataset = SlippyMapTiles(os.path.join(path, 'training', 'images'), transform=train_transform)
n = 0
mean = np.zeros(3, dtype=np.float64)
loader = DataLoader(train_dataset, batch_size=1)
for images, tile in tqdm(loader, desc='Loading', unit='image', ascii=True):
image = torch.squeeze(images)
assert image.size(0) == 3, 'channel first'
image = np.array(image, dtype=np.float64)
n += image.shape[1] * image.shape[2]
mean += np.sum(image, axis=(1, 2))
mean /= n
mean.round(decimals=6, out=mean)
print('mean: {}'.format(mean.tolist()))
std = np.zeros(3, dtype=np.float64)
loader = DataLoader(train_dataset, batch_size=1)
for images, tile in tqdm(loader, desc='Loading', unit='image', ascii=True):
image = torch.squeeze(images)
assert image.size(0) == 3, 'channel first'
image = np.array(image, dtype=np.float64)
difference = np.transpose(image, (1, 2, 0)) - mean
std += np.sum(np.square(difference), axis=(0, 1))
std = np.sqrt(std / (n - 1))
std.round(decimals=6, out=std)
print('std: {}'.format(std.tolist()))
def forward(self, X, X_mask):
#X: [m, Tx] m = batch size, Tx = word count
#print(X.size(), type(X))
m = X.size()[0]
Tx = X.size()[1]
#embedding layer
X = self.embedding(X)
#X: [m, Tx, embedding_dim] m = batch size, Tx = word count
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, self.embedding_dim])
#conv layer
#transpose
X = torch.transpose(X, 1, 2)
#print(X.size(), type(X.data))
X = self.conv(X)
#print(X.size(), type(X.data))
#transpose back
X = torch.transpose(X, 1, 2)
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, Tx, 256])
#maxpool layer
#transpose
X = torch.transpose(X, 1, 2)
X = self.maxpool(X)
#print(X.size(), type(X.data))
#remove dimension
X = X.squeeze()
#print(X.size(), type(X.data))
assert X.size() == torch.Size([m, 256])
#linear
X = self.linear(X)
#X: [m, 1]
#print(X.size(), type(X))
if self.num_classes == 2:
assert X.size() == torch.Size([m, 1])
else:
assert X.size() == torch.Size([m, self.num_classes])
if self.num_classes == 2:
X = torch.squeeze(X)
X = self.sigmoid(X)
#X: [m]
#print(X.size(), type(X))
assert X.size() == torch.Size([m])
return X
else:
return F.softmax(X)
def compute_accuracy(self, prob_cls, gt_cls):
#we only need the detection which >= 0
prob_cls = torch.squeeze(prob_cls)
mask = torch.ge(gt_cls, 0)
#get valid element
valid_gt_cls = torch.masked_select(gt_cls, mask)
valid_prob_cls = torch.masked_select(prob_cls, mask)
size = min(valid_gt_cls.size()[0], valid_prob_cls.size()[0])
prob_ones = torch.ge(valid_prob_cls, 0.6).float()
right_ones = torch.eq(prob_ones, valid_gt_cls.float()).float()
return torch.div(torch.mul(torch.sum(right_ones), float(1.0)), float(size))
def forward(self, x):
x = self.model.conv1(x)
x = self.model.bn1(x)
x = self.model.relu(x)
x = self.model.maxpool(x)
x = self.model.layer1(x)
x = self.model.layer2(x)
x = self.model.layer3(x)
x = self.model.layer4(x)
x = self.model.avgpool(x)
x = torch.squeeze(x)
x = self.classifier(x)
return x
def forward(self, state, action=None):
h = F.relu(self._h1(state.float() / 255.))
h = F.relu(self._h2(h))
h = F.relu(self._h3(h))
h = F.relu(self._h4(h.view(-1, 3136)))
q = self._h5(h)
if action is None:
return q
else:
q_acted = torch.squeeze(q.gather(1, action))
return q_acted
def forward(self, x, y):
x = F.relu(self.conv1(x))
x = F.relu(self.caps_1(x))
b_ij = Variable(torch.Tensor(self.batch_size, 1152, 10, 1, 1), requires_grad=False)
x = self.routing(x, b_ij, self.W,batch_size=self.batch_size,routing_iter=self.routing_iter)
x = torch.squeeze(x, dim=1)
v_length = torch.sqrt(torch.sum(torch.mul(x, x),
dim=2, keepdim=True) + self.epsilon)
v_length = v_length.view(self.batch_size, 10, 1, 1)
masked_v = torch.matmul(torch.squeeze(x).view(self.batch_size, 16, 10), y.view(-1, 10, 1))
vector_j = masked_v.view(self.batch_size, 16)
fc1 = self.recon_fc_1(vector_j)
fc2 = self.recon_fc_2(fc1)
reconstruction = F.sigmoid(self.recon_fc_3(fc2))
return v_length, reconstruction
x = x.view(-1, 160)
x = F.relu(self.fc_debug_0(x))
x = self.fc_debug_1(x)
return F.log_softmax(x)
def forward(self, imgs, motions, mode):
img_size = imgs.size()
motion_size = motions.size()
batch_sz = img_size[0]
seq_len = img_size[1]
imgs = imgs.view(-1, img_size[2], img_size[3], img_size[4])
motions = motions.view(-1, motion_size[2], motion_size[3], motion_size[4])
motions = motions[:, 1:3] # only choose 2 chanel(1,2) for opticflow
for name, module in self.base._modules.items():
if name == 'conv1':
x = module(imgs) + self.conv0(motions)
continue
if name == 'avgpool':
break
x = module(x)
if self.cut_at_pooling:
return x
x = F.avg_pool2d(x, x.size()[2:])
x = x.view(x.size(0), -1)
if mode == 'cnn_rnn':
raw = x.view(batch_sz, seq_len, -1)
if self.has_embedding:
x = self.feat(x)
x = self.feat_bn(x)
if self.dropout > 0:
x = self.drop(x)
if mode == 'cnn_rnn':
# x = x / x.norm(2, 1).expand_as(x)
x = x / x.norm(2, 1).unsqueeze(1).expand_as(x)
x = x.squeeze(1)
x = x.view(batch_sz, seq_len, -1)
return x, raw
elif mode == 'cnn':
x = x / x.norm(2, 1).unsqueeze(1).expand_as(x)
x = x.view(batch_sz, seq_len, -1)
x = torch.squeeze(torch.mean(x, 1), 1)
return x
def forward(model, my_layer, x):
if model.sobel is not None:
x = model.sobel(x)
layer = 1
res = {}
for m in model.features.modules():
if not isinstance(m, nn.Sequential):
x = m(x)
if isinstance(m, nn.ReLU):
if layer == my_layer:
for channel in range(int(x.size()[1])):
key = 'layer' + str(layer) + '-channel' + str(channel)
res[key] = torch.squeeze(x.mean(3).mean(2))[:, channel]
return res
layer = layer + 1
return res
def forward(self, x):
x = self.model.conv1(x)
x = self.model.bn1(x)
x = self.model.relu(x)
x = self.model.maxpool(x)
x = self.model.layer1(x)
x = self.model.layer2(x)
x = self.model.layer3(x)
# x0 n*1024*1*1
x0 = self.model.avgpool(x)
x = self.model.layer4(x)
# x1 n*2048*1*1
x1 = self.model.avgpool(x)
x = torch.cat((x0,x1),1)
x = torch.squeeze(x)
x = self.classifier(x)
return x
def forward(self, feats, conv_feats, lengths=None):
self.processed_batches = self.processed_batches + 1
# print(feats.size(), text_length)
nT = feats.size(0)
nB = feats.size(1)
nC = feats.size(2)
hidden_size = self.hidden_size
input_size = self.input_size
assert (input_size == nC)
# assert (nB == text_length)
if lengths is not None:
max_length, _ = lengths.max(0)
self.max_length = max_length.cpu().data.numpy().item()
num_labels = lengths.data.sum()
# num_steps = text_length.data.max()
# num_labels = text_length.data.sum()
# output_hiddens = Variable(torch.zeros(num_steps, nB, hidden_size).type_as(feats.data))
# output_hiddens = []
output_hiddens = Variable(
torch.zeros(self.max_length, nB, hidden_size).type_as(feats.data))
hidden = Variable(torch.zeros(nB, hidden_size).type_as(feats.data))
# max_locs = torch.zeros(num_steps, nB)
# max_vals = torch.zeros(num_steps, nB)
# for i in range(num_steps):
key_value = 0
probs = torch.zeros([nB, self.max_length, self.num_classes])
tstep = 0
while tstep < self.max_length:
hidden, alpha = self.attention_cell(hidden, feats, conv_feats)
# output_hiddens.append(hidden)
output_hiddens[tstep] = hidden
if self.processed_batches % 500 == 0:
max_val, max_loc = alpha.data.max(1)
# max_locs[i] = max_loc.cpu()
# max_vals[i] = max_val.cpu()
prob = self.generator(hidden)
key = prob.max(1)[1]
key = torch.squeeze(key)
key_value = key.data
if lengths is not None:
new_hiddens = Variable(
torch.zeros(num_labels, hidden_size).type_as(feats.data))
# if probs is None:
# probs = key.data.type(torch.IntTensor)
# else:
# probs = torch.cat([probs, key.data.type(torch.IntTensor)], 0)
start = 0
b = 0
for length in lengths.data:
new_hiddens[start:start +
length] = output_hiddens[0:length, b, :]
start = start + length
b = b + 1
else:
probs[:, tstep, :] = prob.data
if key_value.cpu().numpy().item() == 112:
break
# key_value = key.data.cpu().numpy().item()
# print('key:', key)
tstep += 1
if lengths is not None:
probs = self.generator(new_hiddens)
else:
probs = Variable(probs)
# if self.processed_batches % 500 == 0:
# print('max_locs', max_loc)
# print('max_vals', max_val)
#
return probs
def forward(self, conv_context, actual_tokens=None):
"""
:param conv_context: n_layers, batch, hidden_dim
:return: output_probs: torch.Tensor (seq, batch, corpus_size)
:return: output_tokens: torch.LongTensor (seq, batch)
"""
output_probs = []
output_tokens = []
hidden_state = conv_context
# 0 corresponds to <sos>
batch_size = conv_context.size(1) if type(
conv_context) is not list else conv_context[0].size(1)
next_input_token = Variable(
torch.LongTensor(np.zeros((1, batch_size), dtype=np.int_)))
self.embedding[0].cpu()
#if torch.cuda.is_available():
# next_input_token = next_input_token.cuda() # 1, batch
use_teacher_forcing = actual_tokens is not None
if use_teacher_forcing:
actual_tokens = Variable(actual_tokens.squeeze())
do_continue = True
timestep = 0
while do_continue:
embedded_input = self.embedding[0](
next_input_token).cuda() # 1, batch, embedding_dim
assert len(embedded_input.size()
) == 3, "embedded_input must have 3 dimensions"
# output: 1 x batch x hidden_dim, hidden_state: n_layers, batch, hidden_dim
output, hidden_state = self.rnn(embedded_input, hidden_state)
logits = self.fc(output) # 1 x batch x corpus_size
probs = F.log_softmax(logits, dim=-1) # 1, batch, corpus_size
output_probs.append(probs)
# used for input at the next time step
#top_words = torch.squeeze(torch.topk(probs, 1, -1)[1], 2) # 1 x batch
top_words = torch.squeeze(torch.topk(probs, 1, -1)[1],
0) # 1 x batch
if use_teacher_forcing:
next_input_token = actual_tokens[timestep:timestep +
1].unsqueeze(0)
else:
next_input_token = top_words.cpu()
output_tokens += [top_words]
timestep += 1
# two conditions will make stop this loop
# assumes a batch size of 1 otherwise this gets really complicated
# need to think deeply on how to batch this
#print(torch.squeeze(top_words.data.cpu()).numpy()[0])
if use_teacher_forcing:
if timestep == actual_tokens.size(0):
do_continue = False
del actual_tokens # desperate try to use less memory
elif not use_teacher_forcing:
if top_words.data.cpu().numpy()[0] == self.corpus.eos:
do_continue = False
if timestep == self.max_response_length:
do_continue = False
if not use_teacher_forcing and timestep != self.max_response_length:
print("Decoding ended early:", timestep)
# returns (seq, batch, corpus_size)
return torch.cat(output_probs, 0), torch.cat(output_tokens, 0)
def get_n_tokens_in_sentence(self, sentence):
encodings_dict = self.tokenizer.tokenizer(
sentence, truncation=True, max_length=self.cfg.max_seq_length, padding=False, return_tensors="pt"
)
output = torch.squeeze(encodings_dict['input_ids'])
return len(output) if len(output.size()) > 0 else 0
net.eval()
correct = 0
total_correct = 0
total = len(siamese_dataset)
print(total)
gaussian_mask = get_gaussian_mask().cuda(
) # Only triplet_model 1024 has gaussian mask.
for i in range(total - 1):
anc, pos, negatives = next(dataiter)
print(i)
batch_correct = 0
negatives = torch.squeeze(negatives)
for j in range(len(negatives)):
neg = negatives[j]
neg = neg.unsqueeze(0)
concatenated = torch.cat((anc, pos, neg), 0)
output1, output2, output3 = net(
Variable(anc).cuda() * gaussian_mask,
Variable(pos).cuda() * gaussian_mask,
Variable(neg).cuda() * gaussian_mask)
output1 = torch.unsqueeze(output1, 0) # anc
output2 = torch.unsqueeze(output2, 0) # pos
output3 = torch.unsqueeze(output3, 0) # neg
paths = os.listdir(SAMPLES)
for path in paths:
image_src = cv2.imread(SAMPLES + path)
image_shape = image_src.shape
print(image_shape)
image = cv2.resize(image_src, (480, 352))
image = image / 255.0
image = torch.Tensor(image).to(device)
image = image.permute(2, 0, 1)
image = torch.unsqueeze(image, dim=0)
output = model(image)
output = torch.squeeze(output)
output = output.argmax(dim=0)
output = output.cpu()
output_np = cv2.resize(np.uint8(output),
(image_shape[1], image_shape[0]))
image_seg = np.zeros((image_shape[0], image_shape[1], 3))
image_seg = np.uint8(image_seg)
colors = COLORS
for c in range(CLASS_NUM):
image_seg[:, :, 0] += np.uint8(
(output_np == c)) * np.uint8(colors[c][0])
image_seg[:, :, 1] += np.uint8(
(output_np == c)) * np.uint8(colors[c][1])
def step(
self, Ybar_t: torch.Tensor, dec_state: Tuple[torch.Tensor,
torch.Tensor],
enc_hiddens: torch.Tensor, enc_hiddens_proj: torch.Tensor,
enc_masks: torch.Tensor
) -> Tuple[Tuple, torch.Tensor, torch.Tensor]:
""" Compute one forward step of the LSTM decoder, including the attention computation.
@param Ybar_t (Tensor): Concatenated Tensor of [Y_t o_prev], with shape (b, e + h). The input for the decoder,
where b = batch size, e = embedding size, h = hidden size.
@param dec_state (tuple(Tensor, Tensor)): Tuple of tensors both with shape (b, h), where b = batch size, h = hidden size.
First tensor is decoder's prev hidden state, second tensor is decoder's prev cell.
@param enc_hiddens (Tensor): Encoder hidden states Tensor, with shape (b, src_len, h * 2), where b = batch size,
src_len = maximum source length, h = hidden size.
@param enc_hiddens_proj (Tensor): Encoder hidden states Tensor, projected from (h * 2) to h. Tensor is with shape (b, src_len, h),
where b = batch size, src_len = maximum source length, h = hidden size.
@param enc_masks (Tensor): Tensor of sentence masks shape (b, src_len),
where b = batch size, src_len is maximum source length.
@returns dec_state (tuple (Tensor, Tensor)): Tuple of tensors both shape (b, h), where b = batch size, h = hidden size.
First tensor is decoder's new hidden state, second tensor is decoder's new cell.
@returns combined_output (Tensor): Combined output Tensor at timestep t, shape (b, h), where b = batch size, h = hidden size.
@returns e_t (Tensor): Tensor of shape (b, src_len). It is attention scores distribution.
Note: You will not use this outside of this function.
We are simply returning this value so that we can sanity check
your implementation.
"""
combined_output = None
### YOUR CODE HERE (~3 Lines)
### TODO:
### 1. Apply the decoder to `Ybar_t` and `dec_state`to obtain the new dec_state.
### 2. Split dec_state into its two parts (dec_hidden, dec_cell)
### 3. Compute the attention scores e_t, a Tensor shape (b, src_len).
### Note: b = batch_size, src_len = maximum source length, h = hidden size.
###
### Hints:
### - dec_hidden is shape (b, h) and corresponds to h^dec_t in the PDF (batched)
### - enc_hiddens_proj is shape (b, src_len, h) and corresponds to W_{attProj} h^enc (batched).
### - Use batched matrix multiplication (torch.bmm) to compute e_t.
### - To get the tensors into the right shapes for bmm, you will need to do some squeezing and unsqueezing.
### - When using the squeeze() function make sure to specify the dimension you want to squeeze
### over. Otherwise, you will remove the batch dimension accidentally, if batch_size = 1.
###
### Use the following docs to implement this functionality:
### Batch Multiplication:
### https://pytorch.org/docs/stable/torch.html#torch.bmm
### Tensor Unsqueeze:
### https://pytorch.org/docs/stable/torch.html#torch.unsqueeze
### Tensor Squeeze:
### https://pytorch.org/docs/stable/torch.html#torch.squeeze
dec_state = self.decoder(Ybar_t, dec_state)
(dec_hidden, dec_cell) = dec_state
#e_t = torch.bmm(enc_hiddens_proj, dec_hidden)
e_t = torch.squeeze(
torch.bmm(enc_hiddens_proj, torch.unsqueeze(dec_hidden, 2)), 2)
### END YOUR CODE
# Set e_t to -inf where enc_masks has 1
if enc_masks is not None:
e_t.data.masked_fill_(enc_masks.bool(), -float('inf'))
### YOUR CODE HERE (~6 Lines)
### TODO:
### 1. Apply softmax to e_t to yield alpha_t
### 2. Use batched matrix multiplication between alpha_t and enc_hiddens to obtain the
### attention output vector, a_t.
#$$ Hints:
### - alpha_t is shape (b, src_len)
### - enc_hiddens is shape (b, src_len, 2h)
### - a_t should be shape (b, 2h)
### - You will need to do some squeezing and unsqueezing.
### Note: b = batch size, src_len = maximum source length, h = hidden size.
###
### 3. Concatenate dec_hidden with a_t to compute tensor U_t
### 4. Apply the combined output projection layer to U_t to compute tensor V_t
### 5. Compute tensor O_t by first applying the Tanh function and then the dropout layer.
###
### Use the following docs to implement this functionality:
### Softmax:
### https://pytorch.org/docs/stable/nn.html#torch.nn.functional.softmax
### Batch Multiplication:
### https://pytorch.org/docs/stable/torch.html#torch.bmm
### Tensor View:
### https://pytorch.org/docs/stable/tensors.html#torch.Tensor.view
### Tensor Concatenation:
### https://pytorch.org/docs/stable/torch.html#torch.cat
### Tanh:
### https://pytorch.org/docs/stable/torch.html#torch.tanh
m = nn.Softmax(1)
alpha_t = m(e_t)
#a_t = torch.bmm(alpha_t, enc_hiddens)
a_t = torch.squeeze(
torch.bmm(torch.unsqueeze(alpha_t, 1), enc_hiddens), 1)
U_t = torch.cat((a_t, dec_hidden), 1)
V_t = self.combined_output_projection(U_t)
O_t = self.dropout(torch.tanh(V_t))
### END YOUR CODE
combined_output = O_t
return dec_state, combined_output, e_t
def beam_search(self, src_sent: List[str], beam_size: int=5, max_decoding_time_step: int=70) -> List[Hypothesis]:
""" Given a single source sentence, perform beam search, yielding translations in the target language.
@param src_sent (List[str]): a single source sentence (words)
@param beam_size (int): beam size
@param max_decoding_time_step (int): maximum number of time steps to unroll the decoding RNN
@returns hypotheses (List[Hypothesis]): a list of hypothesis, each hypothesis has two fields:
value: List[str]: the decoded target sentence, represented as a list of words
score: float: the log-likelihood of the target sentence
"""
## A4 code
# src_sents_var = self.vocab.src.to_input_tensor([src_sent], self.device)
## End A4 code
src_sents_var = self.vocab.src.to_input_tensor_char([src_sent], self.device)
src_encodings, dec_init_vec = self.encode(src_sents_var, [len(src_sent)])
src_encodings_att_linear = self.att_projection(src_encodings)
h_tm1 = dec_init_vec
att_tm1 = torch.zeros(1, self.hidden_size, device=self.device)
eos_id = self.vocab.tgt['</s>']
hypotheses = [['<s>']]
hyp_scores = torch.zeros(len(hypotheses), dtype=torch.float, device=self.device)
completed_hypotheses = []
t = 0
while len(completed_hypotheses) < beam_size and t < max_decoding_time_step:
t += 1
hyp_num = len(hypotheses)
exp_src_encodings = src_encodings.expand(hyp_num,
src_encodings.size(1),
src_encodings.size(2))
exp_src_encodings_att_linear = src_encodings_att_linear.expand(hyp_num,
src_encodings_att_linear.size(1),
src_encodings_att_linear.size(2))
## A4 code
# y_tm1 = self.vocab.tgt.to_input_tensor(list([hyp[-1]] for hyp in hypotheses), device=self.device)
# y_t_embed = self.model_embeddings_target(y_tm1)
## End A4 code
y_tm1 = self.vocab.tgt.to_input_tensor_char(list([hyp[-1]] for hyp in hypotheses), device=self.device)
y_t_embed = self.model_embeddings_target(y_tm1)
y_t_embed = torch.squeeze(y_t_embed, dim=0)
x = torch.cat([y_t_embed, att_tm1], dim=-1)
(h_t, cell_t), att_t, _ = self.step(x, h_tm1,
exp_src_encodings, exp_src_encodings_att_linear, enc_masks=None)
# log probabilities over target words
log_p_t = F.log_softmax(self.target_vocab_projection(att_t), dim=-1)
live_hyp_num = beam_size - len(completed_hypotheses)
contiuating_hyp_scores = (hyp_scores.unsqueeze(1).expand_as(log_p_t) + log_p_t).view(-1)
top_cand_hyp_scores, top_cand_hyp_pos = torch.topk(contiuating_hyp_scores, k=live_hyp_num)
prev_hyp_ids = top_cand_hyp_pos / len(self.vocab.tgt)
hyp_word_ids = top_cand_hyp_pos % len(self.vocab.tgt)
new_hypotheses = []
live_hyp_ids = []
new_hyp_scores = []
decoderStatesForUNKsHere = []
for prev_hyp_id, hyp_word_id, cand_new_hyp_score in zip(prev_hyp_ids, hyp_word_ids, top_cand_hyp_scores):
prev_hyp_id = prev_hyp_id.item()
hyp_word_id = hyp_word_id.item()
cand_new_hyp_score = cand_new_hyp_score.item()
hyp_word = self.vocab.tgt.id2word[hyp_word_id]
# Record output layer in case UNK was generated
if hyp_word == "<unk>":
hyp_word = "<unk>"+str(len(decoderStatesForUNKsHere))
decoderStatesForUNKsHere.append(att_t[prev_hyp_id])
new_hyp_sent = hypotheses[prev_hyp_id] + [hyp_word]
if hyp_word == '</s>':
completed_hypotheses.append(Hypothesis(value=new_hyp_sent[1:-1],
score=cand_new_hyp_score))
else:
new_hypotheses.append(new_hyp_sent)
live_hyp_ids.append(prev_hyp_id)
new_hyp_scores.append(cand_new_hyp_score)
if len(decoderStatesForUNKsHere) > 0 and self.charDecoder is not None: # decode UNKs
decoderStatesForUNKsHere = torch.stack(decoderStatesForUNKsHere, dim=0)
decodedWords = self.charDecoder.decode_greedy((decoderStatesForUNKsHere.unsqueeze(0), decoderStatesForUNKsHere.unsqueeze(0)), max_length=21, device=self.device)
assert len(decodedWords) == decoderStatesForUNKsHere.size()[0], "Incorrect number of decoded words"
for hyp in new_hypotheses:
if hyp[-1].startswith("<unk>"):
hyp[-1] = decodedWords[int(hyp[-1][5:])]#[:-1]
if len(completed_hypotheses) == beam_size:
break
live_hyp_ids = torch.tensor(live_hyp_ids, dtype=torch.long, device=self.device)
h_tm1 = (h_t[live_hyp_ids], cell_t[live_hyp_ids])
att_tm1 = att_t[live_hyp_ids]
hypotheses = new_hypotheses
hyp_scores = torch.tensor(new_hyp_scores, dtype=torch.float, device=self.device)
if len(completed_hypotheses) == 0:
completed_hypotheses.append(Hypothesis(value=hypotheses[0][1:],
score=hyp_scores[0].item()))
completed_hypotheses.sort(key=lambda hyp: hyp.score, reverse=True)
return completed_hypotheses
def forward(self, input, target):
"""
Args:
input: (tensor): the shape should be BCH[WD].
where C is the number of classes.
target: (tensor): the shape should be B1H[WD] or BCH[WD].
If the target's shape is B1H[WD], the target that this loss expects should be a class index
in the range [0, C-1] where C is the number of classes.
"""
i = input
t = target
if i.ndim != t.ndim:
raise ValueError(
f"input and target must have the same number of dimensions, got {i.ndim} and {t.ndim}"
)
if target.shape[1] != 1 and target.shape[1] != i.shape[1]:
raise ValueError(
"target must have one channel or have the same shape as the input. "
"If it has one channel, it should be a class index in the range [0, C-1] "
f"where C is the number of classes inferred from 'input': C={i.shape[1]}. "
)
# Change the shape of input and target to
# num_batch x num_class x num_voxels.
if input.dim() > 2:
i = i.view(i.size(0), i.size(1), -1) # N,C,H,W => N,C,H*W
t = t.view(t.size(0), t.size(1),
-1) # N,1,H,W => N,1,H*W or N,C,H*W
else: # Compatibility with classification.
i = i.unsqueeze(2) # N,C => N,C,1
t = t.unsqueeze(2) # N,1 => N,1,1 or N,C,1
# Compute the log proba (more stable numerically than softmax).
logpt = F.log_softmax(i, dim=1) # N,C,H*W
# Keep only log proba values of the ground truth class for each voxel.
if target.shape[1] == 1:
logpt = logpt.gather(1, t.long()) # N,C,H*W => N,1,H*W
logpt = torch.squeeze(logpt, dim=1) # N,1,H*W => N,H*W
# Get the proba
pt = torch.exp(logpt) # N,H*W or N,C,H*W
if self.weight is not None:
self.weight = self.weight.to(i)
# Convert the weight to a map in which each voxel
# has the weight associated with the ground-truth label
# associated with this voxel in target.
at = self.weight[None, :, None] # C => 1,C,1
at = at.expand((t.size(0), -1, t.size(2))) # 1,C,1 => N,C,H*W
if target.shape[1] == 1:
at = at.gather(
1, t.long()) # selection of the weights => N,1,H*W
at = torch.squeeze(at, dim=1) # N,1,H*W => N,H*W
# Multiply the log proba by their weights.
logpt = logpt * at
# Compute the loss mini-batch.
weight = torch.pow(-pt + 1.0, self.gamma)
if target.shape[1] == 1:
loss = torch.mean(-weight * logpt, dim=1) # N
else:
loss = torch.mean(-weight * t * logpt, dim=-1) # N,C
if self.reduction == "sum":
return loss.sum()
if self.reduction == "none":
return loss
if self.reduction == "mean":
return loss.mean()
raise ValueError(f"reduction={self.reduction} is invalid.")
def forward(self, x, top_bc, bottom_bc, left_bc, right_bc):
batch_size = x.shape[0]
# print("x.device: ", x.device)
preprocessed = torch.zeros((x.shape[0], x.shape[2], x.shape[3], 2),
device=x.device)
bc_vector = torch.cat([
torch.squeeze(top_bc)[:, :, 1:-1],
torch.squeeze(bottom_bc)[:, :, 1:-1],
torch.squeeze(left_bc)[:, :, 1:-1],
torch.squeeze(right_bc)[:, :, 1:-1]
],
dim=2)
# print("shapes1: ", preprocessed.shape, bc_vector.shape, self.pre_maps.shape)
bc_vector = bc_vector.unsqueeze(dim=3).unsqueeze(dim=4)
new_maps = self.pre_maps.unsqueeze(dim=0).unsqueeze(dim=1).to(x.device)
weight_map = self.weight_map.unsqueeze(dim=0).unsqueeze(dim=1).to(
x.device)
# print("bc_vector.device: ", bc_vector.device)
# print("new_maps.device: ", new_maps.device)
# print("preprocessed.device: ", preprocessed.device)
preprocessed = bc_vector * new_maps
preprocessed = torch.sum(preprocessed, 2) / weight_map
# preprocessed = preprocessed.permute(0, 3, 1, 2)
# print("shapes4: ", x.shape, preprocessed.shape)
grid = self.get_grid(x.shape, x.device)
pre_data = torch.cat((x, grid, preprocessed), dim=1)
mod_data = torch.cat((x, grid, preprocessed), dim=1)
# modulating branch:
# mod = F.avg_pool2d(self.m_basic1(mod_data),2)
# mod = F.avg_pool2d(self.m_basic2(mod),2)
# mod = F.avg_pool2d(self.m_basic3(mod),2)
# mod = self.m_bc1(mod.reshape((batch_size, -1)))
# # mod1 = self.m_bc2_1(mod).reshape((batch_size, self.width,self.width,1,1))
# # mod2 = self.m_bc2_2(mod).reshape((batch_size, self.width,self.width,1,1))
# mod1 = self.m_bc2_1(mod).reshape((batch_size, 1,1, self.modes1,self.modes2))
# mod2 = self.m_bc2_2(mod).reshape((batch_size, 1,1, self.modes1,self.modes2))
x = pre_data.permute(0, 2, 3, 1)
x = self.fc0_dielectric(x)
x = x.permute(0, 3, 1, 2)
if self.padding > 0:
x = F.pad(x, [0, self.padding, 0, self.padding])
# print(f"a x.shape:{x.shape}")
for i in range(self.num_fourier_layers - 1):
mod = self.encodes[i](x).reshape((batch_size, self.width // 4, -1))
mods = self.attentions[i](mod).reshape(
(-1, 2, self.width, self.width))
mod1 = mods[:, 0, :, :].reshape(
(batch_size, self.width, self.width, 1, 1))
mod2 = mods[:, 1, :, :].reshape(
(batch_size, self.width, self.width, 1, 1))
x1 = self.convs[i](x, mod1, mod2)
x2 = self.ws[i](x)
x = x1 + x2
x = F.leaky_relu(x, negative_slope=self.ALPHA)
# x = F.gelu(x)
x1 = self.convs[-1](x, mod1, mod2)
x2 = self.ws[-1](x)
x = x1 + x2
if self.padding > 0:
x = x[..., :-self.padding, :-self.padding]
x = x.permute(0, 2, 3, 1)
x = self.fc1(x)
x = F.leaky_relu(x, negative_slope=self.ALPHA)
# x = F.gelu(x)
x = self.fc2(x)
x = x.permute(0, 3, 1, 2)
return x
def update(self, memory, update_time):
assert self.K_epochs > 0
# Monte Carlo estimate of rewards:
rewards = []
discounted_reward = 0
for reward, is_terminal in zip(reversed(memory.rewards),
reversed(memory.is_terminals)):
if is_terminal:
discounted_reward = 0
discounted_reward = reward + (self.gamma * discounted_reward)
rewards.insert(0, discounted_reward)
# Normalizing the rewards:
rewards = torch.tensor(rewards)
rewards = (rewards - rewards.mean()) / (rewards.std() + 1e-5)
# convert list to tensor
old_states = torch.squeeze(torch.stack(memory.states)).detach()
old_actions = torch.squeeze(torch.stack(memory.actions)).detach()
old_logprobs = torch.squeeze(torch.stack(memory.logprobs)).detach()
old_means = torch.squeeze(torch.stack(memory.means)).detach()
# construct old distributions
old_dist = MultivariateNormal(old_means.detach(), self.cov_mat)
"""
Hyper parameters for base phi:
use_cv # whether to use control variate
P_epochs # num of epochs to train the new base phi
grad_len # make sure the gradient is not too big
step_size # the step size of the functional gradient (gradient estimator)
phi_hidden_dim # the hidden dimension of the new base phi
phi_lr, betas # learning rate and betas for Adam of base phi
phi_max_bases # max number of base funcs in phi
phi_start_using # num of updates after which we start to use phi
"""
use_cv = 1.
P_epochs = 8000
grad_len = 50.0
step_size = 0.005
phi_hidden_dim = 64
phi_lr = 0.002
phi_betas = (0.9, 0.999)
phi_max_bases = 30
phi_start_using = 0
# update our phi using gradient boosting
if use_cv > 0 and update_time < phi_max_bases:
# new base func
phi = cv.BaseFunc(self.state_dim + self.action_dim, phi_hidden_dim)
# load old func if possible
if len(self.control_variate.bases) > 1:
phi.load_state_dict(
self.control_variate.bases[-1].state_dict())
# optimizer and fixed step size
phi_optim = torch.optim.Adam(phi.parameters(),
lr=phi_lr,
betas=phi_betas)
# get current phi_values, and phi_grad_actions
old_phi_values, old_phi_grad_actions = self.control_variate.get_value(
old_states, old_actions)
# prepare g, [batch_size]
g = torch.Tensor([
utils.calculate_function_grad(self.policy, self.optimizer,
old_phi_values[i],
old_phi_grad_actions[i],
old_states[i], old_actions[i],
rewards[i], self.cov_mat)
for i in range(len(rewards))
]).unsqueeze(-1).detach()
# approximate the functional gradient with nn
for _ in range(P_epochs):
phi_values = phi.forward(old_states, old_actions)
loss = self.MseLoss(g, phi_values)
phi_optim.zero_grad()
loss.backward()
phi_optim.step()
self.control_variate.add_base_func(phi, step_size)
# compute current phi(s, a) and grad_phi w.r.t actions
phi_values, phi_grad_actions = self.control_variate.get_value(
old_states, old_actions)
# when phi is not trained enough, we don't use it
if update_time < phi_start_using:
use_cv = 0
# Optimize policy for K epochs:
for _ in range(self.K_epochs):
"""
Derivation of the update:
Naming: X_grad_Y means the gradient of X w.r.t Y
Assume that a = f(s) = mu + std * noise
f_grad_w = mu_grad_w + (std * noise)_grad_w = mu_grad_w
pi(a|s) = C * e^-(1/2(a - mu)^T*SIGMA^-1*(a - mu))
log_pi(a|s) = logC - (1/2) * (a - mu)^T*SIGMA^-1*(a - mu)
ll_grad_mu = SIGMA^-1 * (a - mu)
ratio = exp(logprobs - old_logprobs)
loss = mu dot (ratio * (ll_grad_mu * (A - phi) + phi_grad_actions)).detach()
loss_grad_w = mu_grad_w dot ratio * (ll_grad_mu * (A - phi) + phi_grad_actions)
= ratio * (mu_grad_w dot ll_grad_mu * (A - phi) + mu_grad_w dot phi_grad_actions)
= ratio * (mu_grad_w dot ll_grad_mu * (A - phi) + f_grad_w dot phi_grad_actions)
= ratio * (ll_grad_w dot (A - phi) + f_grad_w dot phi_grad_actions)
Dimensions:
A - [batch, 1]
mu - [batch, action_dim]
phi - [batch, 1]
ratio - [batch, 1]
ll_grad_mu - [batch, action_dim]
phi_grad_actions - [batch, action_dim]
We're instead using adaptive KL PPO.
"""
# Evaluating old actions and values
action_means, logprobs, dist, state_values, dist_entropy = self.policy.process(
old_states, old_actions)
# calculate KL between the old distributions and new distributions
kl = torch.distributions.kl_divergence(old_dist, dist).mean()
# calculate the gradient of log likelihood of old actions w.r.t the action mean
ll_grad_mu = (old_actions - action_means) / self.action_var
# finding the ratio (pi_theta / pi_theta_old):
ratios = torch.exp(logprobs - old_logprobs)
# calculate advantages
advantages = rewards - state_values
# calculate surrogate loss
surr = ratios.unsqueeze(-1) * (
ll_grad_mu * (advantages.unsqueeze(-1) - use_cv * phi_values) +
use_cv * phi_grad_actions)
# dot product with the action mean to get our surrogate loss
loss1 = -(action_means * surr.detach()).sum(1).mean()
# critic loss
loss2 = self.MseLoss(state_values, rewards)
# kl loss
loss3 = kl * self.beta + self.eta * max(0,
kl - 2.0 * self.kl_targ)**2
# total loss
loss = loss1 + 0.5 * loss2 + loss3 - 0.01 * dist_entropy.mean()
# take gradient step
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# early stopping if KL diverges badly
if kl > self.kl_targ * 4:
break
# adaptive kl penalty:
if kl > self.kl_targ * 2:
self.beta = min(35., 1.5 * self.beta)
elif kl < self.kl_targ / 2:
self.beta = max(1 / 35, self.beta / 1.5)
def save_img(img_tensor, name):
img_tensor = torch.squeeze(img_tensor)
save_image(img_tensor, f'./predictions/{name}.png')
def forward(self,
query_content_id: torch.LongTensor,
query_content_feature: torch.FloatTensor,
mask: torch.Tensor,
seen_content_id: Optional[torch.LongTensor],
seen_content_feature: Optional[torch.FloatTensor],
seen_content_feedback: Optional[torch.FloatTensor],
initial_state: Optional[TensorPair] = None):
# content_emb: (batch, seq, dim)
seen_content_tensor = self._combine_content_feature(content_id=seen_content_id,
content_feature=seen_content_feature,
content_feedback=seen_content_feedback)
if hasattr(self, "encoder_in_proj"):
seen_content_tensor = self.encoder_in_proj(seen_content_tensor)
if self.hparams.get("layer_norm", False):
seen_content_tensor = self.seen_content_layer_norm(seen_content_tensor)
seen_content_tensor = F.relu(seen_content_tensor)
# Apply LSTM
sequence_lengths = self.__class__.get_lengths_from_seq_mask(mask)
clamped_sequence_lengths = sequence_lengths.clamp(min=1)
if self.encoder_type == "vanilla_lstm":
lstm_out, state = self._apply_vanilla_lstm(clamped_sequence_lengths,
initial_state,
seen_content_tensor)
elif self.encoder_type == "augmented_lstm":
lstm_out, state = self._apply_augmented_lstm(clamped_sequence_lengths,
initial_state,
seen_content_tensor)
else:
raise NotImplementedError
if self.hparams.get("layer_norm", False):
lstm_out = self.encoder_layer_norm(lstm_out)
if self.hparams.get("highway_connection", False):
c = torch.sigmoid(self.highway_C(lstm_out))
h = self.highway_H(lstm_out)
lstm_out = (1 - c) * torch.relu(h) + c * torch.relu(seen_content_tensor)
else:
lstm_out = F.relu(lstm_out)
if self.hparams["output_dropout"] > 0:
# lstm_out: (batch, seq, dim)
lstm_out = self.output_dropout(lstm_out)
# query_content_tensor: (batch, seq, dim)
query_content_tensor = self._combine_content_feature(content_id=query_content_id,
content_feature=query_content_feature,
content_feedback=None)
if hasattr(self, "encoder_in_proj"):
query_content_tensor = self.query_proj(query_content_tensor)
# pred_tensor: (batch, seq, dim)
pred_tensor = torch.cat([query_content_tensor, lstm_out], dim=-1)
# y_pred: (batch, seq)
y_pred = torch.squeeze(self.hidden2logit(pred_tensor), dim=-1)
return y_pred, state
def predict_path(self, dataloader, batch_size=1):
results_idx = []
results_predicted = []
results_pose = []
device = self.device
time_thresh = 48
dist_tresh = 500
checkpoint = torch.load('./saved_models/{}'.format(self.load_file),
map_location=self.device)
hyper_params = checkpoint["hyper_params"]
hyper_params['sigma'] = 2
N = self.num_trajectories #number of generated trajectories
model = PECNet(
hyper_params["enc_past_size"], hyper_params["enc_dest_size"],
hyper_params["enc_latent_size"], hyper_params["dec_size"],
hyper_params["predictor_hidden_size"],
hyper_params['non_local_theta_size'],
hyper_params['non_local_phi_size'],
hyper_params['non_local_g_size'], hyper_params["fdim"],
hyper_params["zdim"], hyper_params["nonlocal_pools"],
hyper_params['non_local_dim'], hyper_params["sigma"],
hyper_params["past_length"], hyper_params["future_length"], False)
model = model.double().to(device)
model.load_state_dict(checkpoint["model_state_dict"])
model.eval()
with torch.no_grad():
for b, batch in tqdm(enumerate(dataloader),
total=len(dataloader) // batch_size,
desc='predict'):
scene_images, log_prior, \
agent_masks, \
num_src_trajs, src_trajs, src_lens, src_len_idx, \
num_tgt_trajs, tgt_trajs, tgt_lens, tgt_len_idx, \
tgt_two_mask, tgt_three_mask, \
decode_start_vel, decode_start_pos, scene_id = batch
data = batch
# Detect dynamic batch size
batch_size = scene_images.size(0)
# print(batch_size)
current_path = []
preprocessed = []
agent_num = min(len(data[5]), len(data[9]))
for agent_id in range(agent_num):
past_len = data[5][agent_masks][agent_id]
future_len = data[9][agent_id]
if past_len + future_len == 10:
past = np.array(data[4][agent_masks][agent_id])[::-1]
curr = np.array(
data[-2][agent_masks][agent_id]).reshape(-1, 2)
current_path.append(curr)
future = np.array(data[8][agent_id])
path = np.concatenate((past, curr, future), axis=0)
frame_id = np.arange(0, len(path)).reshape(-1, 1) * 0.5
agent_id_list = np.tile(np.array(agent_id),
(len(path), 1))
xy_coord = np.concatenate(
(frame_id, agent_id_list, path), axis=1)
preprocessed.append(xy_coord)
# print(xy_coord)
####################
### Preprocesing ###
####################
mask_batch = [[0 for i in range(int(agent_num * 1.5))]
for j in range(int(agent_num * 1.5))]
full_dataset = []
full_masks = []
current_batch = []
current_size = 0
data_by_id = {}
for idx in range(len(preprocessed)):
for frame_id, person_id, x, y in preprocessed[idx]:
if person_id not in data_by_id.keys():
data_by_id[person_id] = []
data_by_id[person_id].append(
[person_id, frame_id, x, y])
all_data_dict = data_by_id.copy()
while len(list(data_by_id.keys())) > 0:
related_list = []
curr_keys = list(data_by_id.keys())
current_batch.append((all_data_dict[curr_keys[0]]))
related_list.append(current_size)
current_size += 1
del data_by_id[curr_keys[0]]
for i in range(1, len(curr_keys)):
if social_and_temporal_filter(curr_keys[0],
curr_keys[i],
all_data_dict,
time_thresh, dist_tresh):
current_batch.append((all_data_dict[curr_keys[i]]))
related_list.append(current_size)
current_size += 1
del data_by_id[curr_keys[i]]
mark_similar(mask_batch, related_list)
full_dataset.append(current_batch)
mask_batch = np.array(mask_batch)
full_masks.append(mask_batch[0:len(current_batch),
0:len(current_batch)])
####################
####################
####################
###################################
### Social Dataset Consturction ###
###################################
traj, masks = full_dataset, full_masks
traj_new = []
for t in traj:
t = np.array(t)
t = t[:, :, 2:]
traj_new.append(t)
masks_new = []
for m in masks:
masks_new.append(m)
traj_new = np.array(traj_new)
# print(traj_new.shape)
masks_new = np.array(masks_new)
trajectory_batches = traj_new.copy()
mask_batches = masks_new.copy()
initial_pos_batches = np.array(
initial_position(
trajectory_batches)) #for relative positioning
# print(initial_pos_batches)
###################################
###################################
###################################
#################
### Inference ###
#################
traj = trajectory_batches[0]
mask = mask_batches[0]
initial_pos = initial_pos_batches[0]
traj, mask, initial_pos = torch.DoubleTensor(traj).to(
device), torch.DoubleTensor(mask).to(
device), torch.DoubleTensor(initial_pos).to(device)
x = traj[:, :hyper_params["past_length"], :]
y = traj[:, hyper_params["past_length"]:, :]
y = y.cpu().numpy()
# reshape the data
x = x.contiguous().view(-1, x.shape[1] * x.shape[2])
x = x.to(device)
future = y[:, :-1, :]
dest = y[:, -1, :]
all_l2_errors_dest = []
all_guesses = []
best_of_n = 6
# print(np.reshape(x.cpu().numpy(), (-1, hyper_params["past_length"], 2))[0])
# print(initial_pos.cpu().numpy()[0] *1000)
for index in range(best_of_n):
dest_recon = model.forward(x, initial_pos, device=device)
dest_recon = dest_recon.cpu().numpy()
# print(dest_recon[0])
all_guesses.append(dest_recon)
l2error_sample = np.linalg.norm(dest_recon - dest, axis=1)
all_l2_errors_dest.append(l2error_sample)
all_l2_errors_dest = np.array(all_l2_errors_dest)
all_guesses = np.array(all_guesses)
# average error
l2error_avg_dest = np.mean(all_l2_errors_dest)
# choosing the best guess
indices = np.argmin(all_l2_errors_dest, axis=0)
best_guess_dest = all_guesses[indices,
np.arange(x.shape[0]), :]
# taking the minimum error out of all guess
l2error_dest = np.mean(np.min(all_l2_errors_dest, axis=0))
# back to torch land
best_guess_dest = torch.DoubleTensor(best_guess_dest).to(
device)
predicted_list = []
for goal in all_guesses:
# using the best guess for interpolation
goal = torch.DoubleTensor(goal).to(device)
interpolated_future = model.predict(
x, goal, mask, initial_pos)
interpolated_future = interpolated_future.cpu().numpy()
goal = goal.cpu().numpy()
# final overall prediction
predicted_future = np.concatenate(
(interpolated_future, goal), axis=1)
predicted_future = np.reshape(
predicted_future,
(-1, hyper_params["future_length"], 2))
# - torch.unsqueeze(torch.tensor(initial_pos * 1000), 1).cpu().numpy()
predicted_list.append(predicted_future)
#################
#################
#################
predicted_list = np.transpose(predicted_list, (1, 0, 2, 3))
# interpolated_future = model.predict(x, best_guess_dest, mask, initial_pos)
# interpolated_future = interpolated_future.cpu().numpy()
# best_guess_dest = best_guess_dest.cpu().numpy()
# best_predicted_future = np.concatenate((interpolated_future, best_guess_dest), axis = 1)
# best_predicted_future = np.reshape(best_predicted_future, (-1, hyper_params["future_length"], 2))
# l2error_overall = np.mean(np.linalg.norm(y - best_predicted_future, axis = 2))
# print(l2error_overall)
# print(l2error_dest)
results_idx.append(scene_id)
results_predicted.append(predicted_list)
# results_predicted.append(torch.unsqueeze(torch.tensor(y), 1).cpu().numpy())
results_pose.append(
torch.squeeze(torch.tensor(initial_pos * 1000),
1).cpu().numpy())
return results_idx, results_predicted, results_pose
def __getitem__(self, index):
img0_tuple = random.choice(self.imageFolderDataset.imgs)
negative_images = set()
while len(negative_images) < 32:
# keep looping till a different class image is found. Negative image.
img1_tuple = random.choice(self.imageFolderDataset.imgs)
if img0_tuple[1] == img1_tuple[1]:
continue
else:
negative_images.update([img1_tuple[0]])
negative_images = list(negative_images)
# Selecting positive image.
anchor_image_name = img0_tuple[0].split('/')[-1]
anchor_class_name = img0_tuple[0].split('/')[-2]
all_files_in_class = glob.glob(self.imageFolderDataset.root +
anchor_class_name + '/*')
all_files_in_class = [
x for x in all_files_in_class if x != img0_tuple[0]
]
if len(all_files_in_class) == 0:
positive_image = img0_tuple[0]
else:
positive_image = random.choice(all_files_in_class)
# print(len(positive_image),anchor_class_name,positive_image)
if anchor_class_name != positive_image.split('/')[-2]:
print("Error")
anchor = Image.open(img0_tuple[0])
# negative = Image.open(img1_tuple[0])
positive = Image.open(positive_image)
anchor = anchor.convert("RGB")
# negative = negative.convert("RGB")
positive = positive.convert("RGB")
if self.should_invert:
anchor = PIL.ImageOps.invert(anchor)
positive = PIL.ImageOps.invert(positive)
# negative = PIL.ImageOps.invert(negative)
if self.transform is not None:
anchor = self.transform(anchor)
positive = self.transform(positive)
# negative = self.transform(negative)
negs = []
for i in range(len(negative_images)):
neg_image = Image.open(negative_images[i])
if self.should_invert:
neg_image = PIL.ImageOps.invert(neg_image)
if self.transform is not None:
neg_image = self.transform(neg_image)
negs.append(neg_image)
negatives = torch.squeeze(torch.stack(negs))
return anchor, positive, negatives
def train(self, train, val = None, test = None, verbose = True):
if len(train.Label.unique()) == 2:
self.binary = True
self.config['binary'] = True
lr = self.config['LR']
decay = self.config['decay']
BATCH_SIZE = self.config['batch_size']
train_epoch = self.config['train_epoch']
if 'test_every_X_epoch' in self.config.keys():
test_every_X_epoch = self.config['test_every_X_epoch']
else:
test_every_X_epoch = 40
loss_history = []
self.model = self.model.to(self.device)
# support multiple GPUs
if torch.cuda.device_count() > 1:
if verbose:
print("Let's use " + str(torch.cuda.device_count()) + " GPUs!")
self.model = nn.DataParallel(self.model, dim = 0)
elif torch.cuda.device_count() == 1:
if verbose:
print("Let's use " + str(torch.cuda.device_count()) + " GPU!")
else:
if verbose:
print("Let's use CPU/s!")
# Future TODO: support multiple optimizers with parameters
opt = torch.optim.Adam(self.model.parameters(), lr = lr, weight_decay = decay)
if verbose:
print('--- Data Preparation ---')
params = {'batch_size': BATCH_SIZE,
'shuffle': True,
'num_workers': self.config['num_workers'],
'drop_last': False}
if (self.drug_encoding == "MPNN"):
params['collate_fn'] = mpnn_collate_func
elif self.drug_encoding in ['DGL_GCN', 'DGL_NeuralFP', 'DGL_GIN_AttrMasking', 'DGL_GIN_ContextPred', 'DGL_AttentiveFP']:
params['collate_fn'] = dgl_collate_func
training_generator = data.DataLoader(data_process_loader(train.index.values, train.Label.values, train, **self.config), **params)
if val is not None:
validation_generator = data.DataLoader(data_process_loader(val.index.values, val.Label.values, val, **self.config), **params)
if test is not None:
info = data_process_loader(test.index.values, test.Label.values, test, **self.config)
params_test = {'batch_size': BATCH_SIZE,
'shuffle': False,
'num_workers': self.config['num_workers'],
'drop_last': False,
'sampler':SequentialSampler(info)}
if (self.drug_encoding == "MPNN"):
params_test['collate_fn'] = mpnn_collate_func
elif self.drug_encoding in ['DGL_GCN', 'DGL_NeuralFP', 'DGL_GIN_AttrMasking', 'DGL_GIN_ContextPred', 'DGL_AttentiveFP']:
params_test['collate_fn'] = dgl_collate_func
testing_generator = data.DataLoader(data_process_loader(test.index.values, test.Label.values, test, **self.config), **params_test)
# early stopping
if self.binary:
max_auc = 0
else:
max_MSE = 10000
model_max = copy.deepcopy(self.model)
valid_metric_record = []
valid_metric_header = ["# epoch"]
if self.binary:
valid_metric_header.extend(["AUROC", "AUPRC", "F1"])
else:
valid_metric_header.extend(["MSE", "Pearson Correlation", "with p-value", "Concordance Index"])
table = PrettyTable(valid_metric_header)
float2str = lambda x:'%0.4f'%x
if verbose:
print('--- Go for Training ---')
writer = SummaryWriter()
t_start = time()
iteration_loss = 0
for epo in range(train_epoch):
for i, (v_d, v_p, label) in enumerate(training_generator):
if self.target_encoding == 'Transformer':
v_p = v_p
else:
v_p = v_p.float().to(self.device)
if self.drug_encoding in ["MPNN", 'Transformer', 'DGL_GCN', 'DGL_NeuralFP', 'DGL_GIN_AttrMasking', 'DGL_GIN_ContextPred', 'DGL_AttentiveFP']:
v_d = v_d
else:
v_d = v_d.float().to(self.device)
#score = self.model(v_d, v_p.float().to(self.device))
score = self.model(v_d, v_p)
label = Variable(torch.from_numpy(np.array(label)).float()).to(self.device)
if self.binary:
loss_fct = torch.nn.BCELoss()
m = torch.nn.Sigmoid()
n = torch.squeeze(m(score), 1)
loss = loss_fct(n, label)
else:
loss_fct = torch.nn.MSELoss()
n = torch.squeeze(score, 1)
loss = loss_fct(n, label)
loss_history.append(loss.item())
writer.add_scalar("Loss/train", loss.item(), iteration_loss)
iteration_loss += 1
opt.zero_grad()
loss.backward()
opt.step()
if verbose:
if (i % 100 == 0):
t_now = time()
print('Training at Epoch ' + str(epo + 1) + ' iteration ' + str(i) + \
' with loss ' + str(loss.cpu().detach().numpy())[:7] +\
". Total time " + str(int(t_now - t_start)/3600)[:7] + " hours")
### record total run time
if val is not None:
##### validate, select the best model up to now
with torch.set_grad_enabled(False):
if self.binary:
## binary: ROC-AUC, PR-AUC, F1, cross-entropy loss
auc, auprc, f1, loss, logits = self.test_(validation_generator, self.model)
lst = ["epoch " + str(epo)] + list(map(float2str,[auc, auprc, f1]))
valid_metric_record.append(lst)
if auc > max_auc:
model_max = copy.deepcopy(self.model)
max_auc = auc
if verbose:
print('Validation at Epoch '+ str(epo + 1) + ', AUROC: ' + str(auc)[:7] + \
' , AUPRC: ' + str(auprc)[:7] + ' , F1: '+str(f1)[:7] + ' , Cross-entropy Loss: ' + \
str(loss)[:7])
else:
### regression: MSE, Pearson Correlation, with p-value, Concordance Index
mse, r2, p_val, CI, logits, loss_val = self.test_(validation_generator, self.model)
lst = ["epoch " + str(epo)] + list(map(float2str,[mse, r2, p_val, CI]))
valid_metric_record.append(lst)
if mse < max_MSE:
model_max = copy.deepcopy(self.model)
max_MSE = mse
if verbose:
print('Validation at Epoch '+ str(epo + 1) + ' with loss:' + str(loss_val.item())[:7] +', MSE: ' + str(mse)[:7] + ' , Pearson Correlation: '\
+ str(r2)[:7] + ' with p-value: ' + str(f"{p_val:.2E}") +' , Concordance Index: '+str(CI)[:7])
writer.add_scalar("valid/mse", mse, epo)
writer.add_scalar("valid/pearson_correlation", r2, epo)
writer.add_scalar("valid/concordance_index", CI, epo)
writer.add_scalar("Loss/valid", loss_val.item(), iteration_loss)
table.add_row(lst)
else:
model_max = copy.deepcopy(self.model)
# load early stopped model
self.model = model_max
if val is not None:
#### after training
prettytable_file = os.path.join(self.result_folder, "valid_markdowntable.txt")
with open(prettytable_file, 'w') as fp:
fp.write(table.get_string())
if test is not None:
if verbose:
print('--- Go for Testing ---')
if self.binary:
auc, auprc, f1, loss, logits = self.test_(testing_generator, model_max, test = True)
test_table = PrettyTable(["AUROC", "AUPRC", "F1"])
test_table.add_row(list(map(float2str, [auc, auprc, f1])))
if verbose:
print('Validation at Epoch '+ str(epo + 1) + ' , AUROC: ' + str(auc)[:7] + \
' , AUPRC: ' + str(auprc)[:7] + ' , F1: '+str(f1)[:7] + ' , Cross-entropy Loss: ' + \
str(loss)[:7])
else:
mse, r2, p_val, CI, logits, loss_test = self.test_(testing_generator, model_max)
test_table = PrettyTable(["MSE", "Pearson Correlation", "with p-value", "Concordance Index"])
test_table.add_row(list(map(float2str, [mse, r2, p_val, CI])))
if verbose:
print('Testing MSE: ' + str(mse) + ' , Pearson Correlation: ' + str(r2)
+ ' with p-value: ' + str(f"{p_val:.2E}") +' , Concordance Index: '+str(CI))
np.save(os.path.join(self.result_folder, str(self.drug_encoding) + '_' + str(self.target_encoding)
+ '_logits.npy'), np.array(logits))
######### learning record ###########
### 1. test results
prettytable_file = os.path.join(self.result_folder, "test_markdowntable.txt")
with open(prettytable_file, 'w') as fp:
fp.write(test_table.get_string())
### 2. learning curve
fontsize = 16
iter_num = list(range(1,len(loss_history)+1))
plt.figure(3)
plt.plot(iter_num, loss_history, "bo-")
plt.xlabel("iteration", fontsize = fontsize)
plt.ylabel("loss value", fontsize = fontsize)
pkl_file = os.path.join(self.result_folder, "loss_curve_iter.pkl")
with open(pkl_file, 'wb') as pck:
pickle.dump(loss_history, pck)
fig_file = os.path.join(self.result_folder, "loss_curve.png")
plt.savefig(fig_file)
if verbose:
print('--- Training Finished ---')
writer.flush()
writer.close()
def train():
args = parse_args()
torch.cuda.set_device(args.local_rank)
dist.init_process_group(backend='nccl',
init_method='tcp://127.0.0.1:33271',
world_size=torch.cuda.device_count(),
rank=args.local_rank)
setup_logger(respth)
## dataset
n_classes = 19
n_img_per_gpu = 8
n_workers = 4
cropsize = [1024, 1024]
ds = CityScapes('./data', cropsize=cropsize, mode='train')
sampler = torch.utils.data.distributed.DistributedSampler(ds)
dl = DataLoader(ds,
batch_size=n_img_per_gpu,
shuffle=False,
sampler=sampler,
num_workers=n_workers,
pin_memory=True,
drop_last=True)
## model
ignore_idx = 255
net = BiSeNet(n_classes=n_classes)
net.cuda()
net.train()
net = nn.parallel.DistributedDataParallel(net,
device_ids=[
args.local_rank,
],
output_device=args.local_rank)
score_thres = 0.7
n_min = n_img_per_gpu * cropsize[0] * cropsize[1] // 16
criteria_p = OhemCELoss(thresh=score_thres,
n_min=n_min,
ignore_lb=ignore_idx)
criteria_16 = OhemCELoss(thresh=score_thres,
n_min=n_min,
ignore_lb=ignore_idx)
criteria_32 = OhemCELoss(thresh=score_thres,
n_min=n_min,
ignore_lb=ignore_idx)
## optimizer
momentum = 0.9
weight_decay = 5e-4
lr_start = 1e-2
max_iter = 80000
power = 0.9
warmup_steps = 1000
warmup_start_lr = 1e-5
optim = Optimizer(model=net.module,
lr0=lr_start,
momentum=momentum,
wd=weight_decay,
warmup_steps=warmup_steps,
warmup_start_lr=warmup_start_lr,
max_iter=max_iter,
power=power)
## train loop
msg_iter = 50
loss_avg = []
st = glob_st = time.time()
diter = iter(dl)
epoch = 0
for it in range(max_iter):
try:
im, lb = next(diter)
if not im.size()[0] == n_img_per_gpu: raise StopIteration
except StopIteration:
epoch += 1
sampler.set_epoch(epoch)
diter = iter(dl)
im, lb = next(diter)
im = im.cuda()
lb = lb.cuda()
H, W = im.size()[2:]
lb = torch.squeeze(lb, 1)
optim.zero_grad()
out, out16, out32 = net(im)
lossp = criteria_p(out, lb)
loss2 = criteria_16(out16, lb)
loss3 = criteria_32(out32, lb)
loss = lossp + loss2 + loss3
loss.backward()
optim.step()
loss_avg.append(loss.item())
## print training log message
if (it + 1) % msg_iter == 0:
loss_avg = sum(loss_avg) / len(loss_avg)
lr = optim.lr
ed = time.time()
t_intv, glob_t_intv = ed - st, ed - glob_st
eta = int((max_iter - it) * (glob_t_intv / it))
eta = str(datetime.timedelta(seconds=eta))
msg = ', '.join([
'it: {it}/{max_it}',
'lr: {lr:4f}',
'loss: {loss:.4f}',
'eta: {eta}',
'time: {time:.4f}',
]).format(it=it + 1,
max_it=max_iter,
lr=lr,
loss=loss_avg,
time=t_intv,
eta=eta)
logger.info(msg)
loss_avg = []
st = ed
## dump the final model
save_pth = osp.join(respth, 'model_final.pth')
net.cpu()
state = net.module.state_dict() if hasattr(net,
'module') else net.state_dict()
if dist.get_rank() == 0: torch.save(state, save_pth)
logger.info('training done, model saved to: {}'.format(save_pth))
tree = [[] for i in range(levels + 1)]
cutdim = [[] for i in range(levels)]
tree[0].append(point_set)
for level in range(levels):
for item in tree[level]:
left_ps, right_ps, dim = split_ps(item)
tree[level+1].append(left_ps)
tree[level+1].append(right_ps)
cutdim[level].append(dim)
cutdim[level].append(dim)
else:
tree[0] = [point_set]
for level in range(levels):
for pos, item in enumerate(tree[level]):
split_ps_reuse(item, level, pos, tree, cutdim)
#print level, pos
cutdim_v = [(torch.from_numpy(np.array(item).astype(np.int64))) for item in cutdim]
points = torch.stack(tree[-1])
points_v = Variable(torch.unsqueeze(torch.squeeze(points), 0)).transpose(2,1).cuda()
pred = net(points_v, cutdim_v)
pred_choice = pred.data.max(1)[1]
correct = pred_choice.eq(target.data).cpu().sum()
corrects.append(correct)
print("%d/%d , %f" %(j, len(d), sum(corrects)/ float(len(corrects))))
print(sum(corrects)/ float(len(d)))
for p in D_DC.parameters():
p.requires_grad = False
optim_A = Optimizer.get_optimizer([A], learning_rate=LEARNING_RATE_A)
optim_z = Optimizer.get_optimizer([z], learning_rate=LEARNING_RATE_z)
mseloss = torch.nn.MSELoss(size_average=False)
l1loss = torch.nn.L1Loss(size_average=False)
for epoch in range(EPOCHS):
MUAPs = G_DC(z)
MUAPs = torch.matmul(MUAPs, coeff_matrix) # 对每个MUAPs进行100Hz的高通滤波
MUAPs_logits = D_DC(MUAPs)
MUAPs = torch.squeeze(MUAPs)
if GEN_SEARCH_NUM == 1:
MUAPs = torch.unsqueeze(MUAPs, 0)
if USE_ABS:
reconstruct_EMG = torch.matmul(A, MUAPs) # torch.abs
else:
reconstruct_EMG = torch.matmul(A, MUAPs)
if batch_size > 1:
penalty_A = torch.mean(torch.std(A, dim=0)) - torch.mean(torch.abs(A)) # 第一项希望A尽可能相同，第二项希望A的值不要是零
#loss = LAMBDA * mseloss(reconstruct_EMG, EMG_mvc) - LAMBDA_1 * torch.mean(MUAPs_logits) + LAMBDA_2 * penalty_A
loss = LAMBDA * l1loss(reconstruct_EMG, EMG_mvc) - LAMBDA_1 * torch.mean(MUAPs_logits) + LAMBDA_2 * penalty_A
else:
penalty_A = - torch.mean(torch.abs(A)) # 希望A的值越大越好
#loss = LAMBDA * mseloss(reconstruct_EMG, EMG_mvc) - LAMBDA_1 * torch.mean(MUAPs_logits) + LAMBDA_2 * penalty_A
def decode(self, enc_hiddens: torch.Tensor, enc_masks: torch.Tensor,
dec_init_state: Tuple[torch.Tensor, torch.Tensor],
target_padded: torch.Tensor) -> torch.Tensor:
"""Compute combined output vectors for a batch.
@param enc_hiddens (Tensor): Hidden states (b, src_len, h*2), where
b = batch size, src_len = maximum source sentence length, h = hidden size.
@param enc_masks (Tensor): Tensor of sentence masks (b, src_len), where
b = batch size, src_len = maximum source sentence length.
@param dec_init_state (tuple(Tensor, Tensor)): Initial state and cell for decoder
@param target_padded (Tensor): Gold-standard padded target sentences (tgt_len, b), where
tgt_len = maximum target sentence length, b = batch size.
@returns combined_outputs (Tensor): combined output tensor (tgt_len, b, h), where
tgt_len = maximum target sentence length, b = batch_size, h = hidden size
"""
# Chop of the <END> token for max length sentences.
target_padded = target_padded[:-1]
# Initialize the decoder state (hidden and cell)
dec_state = dec_init_state
# Initialize previous combined output vector o_{t-1} as zero
batch_size = enc_hiddens.size(0)
o_prev = torch.zeros(batch_size, self.hidden_size, device=self.device)
# Initialize a list we will use to collect the combined output o_t on each step
combined_outputs = []
### YOUR CODE HERE (~9 Lines)
### TODO:
### 1. Apply the attention projection layer to `enc_hiddens` to obtain `enc_hiddens_proj`,
### which should be shape (b, src_len, h),
### where b = batch size, src_len = maximum source length, h = hidden size.
### This is applying W_{attProj} to h^enc, as described in the PDF.
### 2. Construct tensor `Y` of target sentences with shape (tgt_len, b, e) using the target model embeddings.
### where tgt_len = maximum target sentence length, b = batch size, e = embedding size.
### 3. Use the torch.split function to iterate over the time dimension of Y.
### Within the loop, this will give you Y_t of shape (1, b, e) where b = batch size, e = embedding size.
### - Squeeze Y_t into a tensor of dimension (b, e).
### - Construct Ybar_t by concatenating Y_t with o_prev.
### - Use the step function to compute the the Decoder's next (cell, state) values
### as well as the new combined output o_t.
### - Append o_t to combined_outputs
### - Update o_prev to the new o_t.
### 4. Use torch.stack to convert combined_outputs from a list length tgt_len of
### tensors shape (b, h), to a single tensor shape (tgt_len, b, h)
### where tgt_len = maximum target sentence length, b = batch size, h = hidden size.
###
### Note:
### - When using the squeeze() function make sure to specify the dimension you want to squeeze
### over. Otherwise, you will remove the batch dimension accidentally, if batch_size = 1.
###
### Use the following docs to implement this functionality:
### Zeros Tensor:
### https://pytorch.org/docs/stable/torch.html#torch.zeros
### Tensor Splitting (iteration):
### https://pytorch.org/docs/stable/torch.html#torch.split
### Tensor Dimension Squeezing:
### https://pytorch.org/docs/stable/torch.html#torch.squeeze
### Tensor Concatenation:
### https://pytorch.org/docs/stable/torch.html#torch.cat
### Tensor Stacking:
### https://pytorch.org/docs/stable/torch.html#torch.stack
enc_hiddens_proj = self.att_projection(enc_hiddens)
Y = self.model_embeddings.target(target_padded)
splitted = torch.split(Y, 1)
for Y_t in splitted:
Y_t = torch.squeeze(Y_t)
Ybar_t = torch.cat((Y_t, o_prev), dim=1)
dec_state, o_t, _ = self.step(Ybar_t, dec_state, enc_hiddens,
enc_hiddens_proj, enc_masks)
combined_outputs.append(o_t)
o_prev = o_t
combined_outputs = torch.stack(combined_outputs)
### END YOUR CODE
return combined_outputs
def squeeze(input, dim):
return th.squeeze(input, dim)
def __next__(self):
out, label = super().__next__()
label = torch.squeeze(label)
return out[0], label.long()
grid = [1, 15] + list(range(30, 451, 30)) + [455]
no_replicates = 10
SMSE_results = np.zeros([len(grid), no_replicates])
SNLP_results = np.zeros([len(grid), no_replicates])
for grid_index, no_inducing in enumerate(grid):
for replicate_index in range(no_replicates):
varGP = variational_GP(x_train_normalised.data.numpy(),
np.expand_dims(
y_train_normalised.data.numpy(), 1),
no_inducing=no_inducing)
varGP.optimize_parameters(1000, 'Adam', learning_rate=0.01)
pred_mean, pred_covar = varGP.joint_posterior_predictive(
x_test_normalised.data.numpy(), noise=True)
pred_mean = torch.squeeze(pred_mean * train_sd[-1] +
train_mean[-1])
pred_var = torch.diag(pred_covar)
pred_var = pred_var * (train_sd[-1])**2
SNLP_varGP = SNLP(pred_mean, pred_var, y_test, train_mean[-1],
train_sd[-1])
SMSE_varGP = SMSE(torch.Tensor(pred_mean), y_test)
SMSE_results[grid_index, replicate_index] = SMSE_varGP
SNLP_results[grid_index, replicate_index] = SNLP_varGP
np.savetxt('SMSE_results.tsv', SMSE_results, delimiter='\t')
np.savetxt('SNLP_results.tsv', SNLP_results, delimiter='\t')
if (print_timing_statements):
print('Sorting time: ' + str(end - start))
start = time.time()
optimizer.zero_grad()
predictions = model(batch_reviews_vec_sorted)
end = time.time()
if (print_timing_statements):
print('Predictions time: ' + str(end - start))
start = time.time()
loss = loss_function(
torch.squeeze(predictions),
torch.tensor(batch_labels_sorted, dtype=torch.float))
loss.backward()
end = time.time()
if (print_timing_statements):
print('Backprop time: ' + str(end - start))
start = time.time()
# clip gradient
torch.nn.utils.clip_grad_norm_(model.parameters(),
5.0) # using 5.0 as default from HW4
optimizer.step()
def forward(self, obs):
return torch.squeeze(self.v_net(obs),
-1) # Critical to ensure v has right shape.
def train(config, model):
reader = task_reader.RelationExtractionMultiCLSReader(
vocab_path="ERNIE_pretrain/vocab.txt",
label_map_config='data/relation2label.json',
spo_label_map_config='data/label2relation.json',
max_seq_len=256,
do_lower_case=True,
in_tokens=False,
random_seed=1,
task_id=0,
num_labels=112)
train_iter = reader.data_generator(input_file='data/train_data.json',
batch_size=16,
epoch=20,
shuffle=True,
phase="train")
test_iter = reader.data_generator(input_file='data/dev_data.json',
batch_size=16,
epoch=1,
shuffle=False,
phase='test')
output, f1 = evaluate(test_iter, model)
return
num_train_examples = reader.get_num_examples('data/train_data.json')
train_steps = 20 * num_train_examples // 16
model.train()
param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [{
'params':
[p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
'weight_decay':
0.01
}, {
'params':
[p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
'weight_decay':
0.0
}]
# optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
optimizer = BertAdam(optimizer_grouped_parameters,
lr=config.learning_rate,
warmup=0.05,
t_total=train_steps) # epoch included
total_batch = 0 # 记录进行到多少batch
dev_best_loss = float('inf')
logger = logging.getLogger(__name__)
last_improve = 0 # 记录上次验证集loss下降的batch数
flag = False # 记录是否很久没有效果提升
model.train()
best_f1 = 0
if True:
for i, it in enumerate(train_iter()):
sys.stdout.flush()
sent = torch.squeeze(torch.tensor(it[0])).cuda()
mask = torch.squeeze(torch.tensor(it[4])).cuda()
label = torch.squeeze(torch.tensor(it[5])).cuda()
lens = torch.squeeze(torch.tensor(it[6])).cuda()
outputs = model(sent, mask)
logits = torch.flatten(outputs, start_dim=0, end_dim=1)
labels = torch.flatten(label, start_dim=0, end_dim=1)
input_mask = torch.flatten(mask, start_dim=0, end_dim=1)
log_logits = torch.log(logits)
log_logits_neg = torch.log(1 - logits)
ce_loss = 0. - labels * log_logits - (1 - labels) * log_logits_neg
ce_loss = torch.mean(ce_loss)
ce_loss = ce_loss * input_mask
loss = torch.mean(x=ce_loss)
lod = logits.cpu().detach().numpy()
lab = labels.cpu().detach().numpy()
msk = input_mask.cpu().numpy().tolist()
num_correct, num_total = calculate_acc(lod, lab, msk)
model.zero_grad()
loss.backward()
optimizer.step()
if (i > 0 and i % 10700 == 0):
dev_iter = reader.data_generator(
input_file='data/dev_data.json',
batch_size=16,
epoch=1,
shuffle=False,
phase="train")
output, f1 = evaluate(dev_iter, model)
if (f1 > best_f1):
torch.save(model.state_dict(), config.save_path)
best_f1 = f1
def evaluate(dev_iter, model):
spo_label_map = json.load(open("data/label2relation.json"))
examples = []
model.eval()
f = open("data/dev_data.json", 'r', encoding="utf8")
for line in f.readlines():
examples.append(json.loads(line))
f.close()
tp, fp, fn = 0, 0, 0
output_file = open("test_out.txt", 'w')
with torch.no_grad():
for index, it in enumerate(dev_iter()):
sys.stdout.flush()
start = time.clock()
# prepare fetched batch data: unlod etc.
# [logits, labels,example_index_list, tok_to_orig_start_index_list, tok_to_orig_end_index_list ] = it
example_index_list = it[7]
tok_to_orig_start_index_list = it[8]
tok_to_orig_end_index_list = it[9]
example_index_list = np.array(example_index_list).astype(
int) - 100000
sent = torch.squeeze(torch.tensor(it[0])).cuda()
mask = torch.squeeze(torch.tensor(it[4])).cuda()
label = it[5]
lens = it[6]
outs = model(sent, mask)
logits = outs.cpu().detach().numpy()
# no .lod() in torch, use mask/seq length instead
for i in range(np.size(logits, 0)):
start2 = time.clock()
# prepare prediction results for each example
example_index = example_index_list[i]
example = examples[example_index]
tok_to_orig_start_index = tok_to_orig_start_index_list[i]
tok_to_orig_end_index = tok_to_orig_end_index_list[i]
inference_tmp = logits[i]
labels_tmp = label[i]
l = lens[i]
# some simple post process
inference_tmp = post_process(inference_tmp, l)
# logits -> classification results
inference_tmp[inference_tmp >= 0.5] = 1
inference_tmp[inference_tmp < 0.5] = 0
predict_result = []
for j, token in enumerate(inference_tmp):
if (j >= l):
break
predict_result.append(np.argwhere(token == 1).tolist())
# format prediction into spo, calculate metric
formated_result = format_output(example, predict_result,
spo_label_map,
tok_to_orig_start_index,
tok_to_orig_end_index, l)
tp_tmp, fp_tmp, fn_tmp = calculate_metric(
example['spo_list'], formated_result['spo_list'], l)
# formated_result['text']=example['text']
# fk = json.dump(formated_result,output_file,ensure_ascii=False)
# print('\n',file=output_file,end='')
tp += tp_tmp
fp += fp_tmp
fn += fn_tmp
p = tp / (tp + fp) if tp + fp != 0 else 0
r = tp / (tp + fn) if tp + fn != 0 else 0
f = 2 * p * r / (p + r) if p + r != 0 else 0
return "[evaluation] precision: %f, recall: %f, f1: %f" % (p, r, f), f
def run_epoch(model, data, is_train=False, lr=1.0):
"""Runs the model on the given data."""
if is_train:
model.train()
else:
model.eval()
epoch_size = ((len(data) // model.batch_size) - 1) // model.num_steps
start_time = time.time()
hidden = model.init_hidden()
costs = 0.0
iters = 0
for step, (x, y) in enumerate(
reader.ptb_iterator(data, model.batch_size, model.num_steps)):
inputs = Variable(
torch.from_numpy(x.astype(np.int64)).transpose(
0, 1).contiguous()).cuda()
model.zero_grad()
hidden = repackage_hidden(hidden)
num_steps_time, bs = inputs.size()
indices = np.random.permutation(bs)
targets = Variable(
torch.from_numpy(y.astype(np.int64)).transpose(
0, 1).contiguous()).cuda()
if is_train:
#alpha = 0.1
lam = np.random.beta(args.mixup_alpha, args.mixup_alpha)
#lam = np.random.uniform(0.95, 1.0)
lam = Variable(
torch.from_numpy(np.array([lam]).astype('float32')).cuda())
targets = targets.permute(1, 0)
target_shuffled = targets[indices]
targets = targets.permute(1, 0).contiguous()
target_shuffled = target_shuffled.permute(1, 0).contiguous()
tt_shuffled = torch.squeeze(
target_shuffled.view(-1, model.batch_size * model.num_steps))
targets = Variable(
torch.from_numpy(y.astype(np.int64)).transpose(
0, 1).contiguous()).cuda()
tt = torch.squeeze(targets.view(-1,
model.batch_size * model.num_steps))
if is_train:
outputs, hidden = model(inputs, hidden, is_train, indices, lam)
loss = lam * criterion(outputs.view(
-1, model.vocab_size), tt) + (1 - lam) * criterion(
outputs.view(-1, model.vocab_size), tt_shuffled)
else:
outputs, hidden = model(inputs, hidden, False, None, None)
loss = criterion(outputs.view(-1, model.vocab_size), tt)
costs += loss.data[0] * model.num_steps
iters += model.num_steps
if is_train:
loss.backward()
torch.nn.utils.clip_grad_norm(model.parameters(), 0.25)
for p in model.parameters():
p.data.add_(-lr, p.grad.data)
if step % (epoch_size // 10) == 10:
print("{} perplexity: {:8.2f} speed: {} wps".format(
step * 1.0 / epoch_size, np.exp(costs / iters),
iters * model.batch_size / (time.time() - start_time)))
return np.exp(costs / iters)
try:
if ret:
frame = cv2.resize(frame, (640, 480))
frame = (frame[:, :, ::-1] / 255.0).astype(
np.float32
) #convert BGR to RGB, convert to 0-1 range and cast to float32
frame_tensor = torch.unsqueeze(torch.from_numpy(frame),
0).permute(0, 3, 1, 2)
# add batch dimension and convert to NCHW format
tensor_in = preprocess(frame_tensor) #normalize
tensor_in = tensor_in.to(device) #send to GPU
tensor_out = net(tensor_in) #stylized tensor
tensor_out = torch.squeeze(tensor_out).permute(
1, 2, 0
) #remove batch dimension and convert to HWC (opencv format)
stylized_frame = (
255 * (tensor_out.to('cpu').detach().numpy())).astype(
np.uint8) #convert to 0-255 range and cast as uint8
stylized_frame = cv2.resize(stylized_frame, (1280, 720))
else:
stylized_frame = dummyframe #if camera cannot be read, blank white image will be shown
vcam.send(stylized_frame)
#write to ffmpeg pipeline which in turn writes to virtual camera that can be accessed by zoom/skype/teams
ret, frame = src.read()
except KeyboardInterrupt:
print('Received stop command')
def test(model, test_loader, epoch, test_list, save_dir):
model.eval()
if not isdir(save_dir):
os.makedirs(save_dir)
for idx, image in enumerate(test_loader):
image = image.cuda()
_, _, H, W = image.shape
results = model(image)
result = torch.squeeze(results[-1].detach()).cpu().numpy()
results_all = torch.zeros((len(results), 1, H, W))
################# out code ####################
print("Start of our Code")
final = result.copy()
# Maybe we need step of emphzise edges and remove the contains
ret, thresh1 = cv2.threshold(255*final, 100, 255, cv2.THRESH_BINARY)
# cv2.imshow("thresh1", thresh1)
# cv2.waitKey(0)
edges = cv2.Canny(np.uint8(thresh1), 100, 200)
# cv2.imshow("Edges of fusion image", edges)
# cv2.waitKey(0)
# it's take the points as sample of maskRCNN mask points
refPt.clear()
for dir in [Direction.Down,Direction.Left,Direction.Up,Direction.Right]:
print("Start New Snapping")
cv2.namedWindow("image")
cv2.setMouseCallback("image", pick_pionts)
cv2.imshow("image", edges)
cv2.waitKey(0)
print("We have Down Snapping now ",len(refPt))
snap_nearestEdge(refPt,edges,dir)
refPt.clear()
# print("Start New Snapping")
# refPt.clear()
# cv2.namedWindow("image")
# cv2.setMouseCallback("image", pick_pionts)
# cv2.imshow("image", edges)
# cv2.waitKey(0)
# print("We have left Snapping now ", len(refPt))
# snap_nearestEdge(refPt, edges, Direction.Left)
#
# print("Start New Snapping")
# refPt.clear()
# cv2.namedWindow("image")
# cv2.setMouseCallback("image", pick_pionts)
# cv2.imshow("image", edges)
# cv2.waitKey(0)
# print("We have left Snapping now ", len(refPt))
# snap_nearestEdge(refPt, edges, Direction.Up)
#
# print("Start New Snapping")
# refPt.clear()
# cv2.namedWindow("image")
# cv2.setMouseCallback("image", pick_pionts)
# cv2.imshow("image", edges)
# cv2.waitKey(0)
# print("We have left Snapping now ", len(refPt))
# snap_nearestEdge(refPt, edges, Direction.Right)
# i=2
# while(i>0):
# kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (i, i))
# closing = cv2.morphologyEx(thresh1, cv2.MORPH_CLOSE, kernel)
# cv2.imshow("closing", closing)
# cv2.waitKey(0)
# print("opening",closing)
# edges = cv2.Canny(np.uint8(closing), 100, 200)
# cv2.imshow("Edges of fusion image",edges)
# cv2.waitKey(0)
# thresh1= closing.copy()
# i=i+1
print("End of our Code")
for i in range(len(results)):
results_all[i, 0, :, :] = results[i]
filename = splitext(test_list[idx])[0]
torchvision.utils.save_image(1-results_all, join(save_dir, "%s.jpg" % filename))
result = Image.fromarray((result * 255).astype(np.uint8))
result.save(join(save_dir, "%s.png" % filename))
print("Running test [%d/%d]" % (idx + 1, len(test_loader)))
os.mkdir(expert_data_file)
expert_data = {"obs":[], "action":[]}
obs = env.reset()
state = state_norm(obs, update=False)
rew = 0
rew_list = []
epi = 0
while epi <= args.expert_num:
# env.render()
expert_data["obs"].append(obs)
state_tensor = torch.tensor(state, dtype=torch.float32, device=device).unsqueeze(0)
# a, var = actor(state_tensor)
a = actor.select_action(state_tensor)
# pi = actor.log_pi(state_tensor, a)
a = torch.squeeze(a, 0).detach().cpu().numpy()
a = np.clip(a, -1, 1)
expert_data["action"].append(a)
obs, r, d, _ = env.step(a)
rew += r
if d:
rew_list.append(rew)
epi += 1
print("reward", rew)
# if epi % 10 == 0:
# print("teset_", np.mean(rew_list))
# rew_list = []
obs = env.reset()
rew = 0
