def forward(self, predict: tensor, target: tensor):
mse_loss = self.mse_loss(predict, target)
l1_loss = self.l1_loss(predict, target)
self.prediction = predict.detach().cpu()
self.target = target.detach().cpu()
ssim = 1 - self.ssim_score()
return mse_loss + l1_loss + ssim
def official_iou(output: torch.tensor,
target: torch.tensor,
numClass: int = NUM_CLASSES) -> tuple:
"""
Official metric from
https://github.com/CSAILVision/sceneparsing/blob/master/evaluationCode/utils_eval.py
This function takes the prediction and label of a single image, returns intersection and union areas for each class
To compute over many images do:
for i in range(Nimages):
(area_intersection[:,i], area_union[:,i]) = intersectionAndUnion(imPred[i], imLab[i])
IoU = 1.0 * np.sum(area_intersection, axis=1) / np.sum(np.spacing(1)+area_union, axis=1)
"""
#output = torch.argmax(output, dim=1)
output = np.asarray(output.detach().cpu().numpy())
target = np.asarray(target.detach().cpu().numpy())
# Remove classes from unlabeled pixels in gt image.We should not penalize detections in unlabeled portions
output = output * (target > 0)
# Compute area intersection:
intersection = output * (output == target)
(area_intersection, _) = np.histogram(intersection,
bins=numClass,
range=(1, numClass))
# Compute area union:
(area_pred, _) = np.histogram(output, bins=numClass, range=(1, numClass))
(area_lab, _) = np.histogram(target, bins=numClass, range=(1, numClass))
area_union = area_pred + area_lab - area_intersection
return (area_intersection, area_union)
def decode_mlm_batch_output(
token_ids: torch.
tensor, # integer array with shape [batch size, seq length]
logits: torch.tensor,
utterances: List[List[str]],
mask_token_id: int, # token_id corresponding to [MASK]
) -> List[List[str]]:
"""
:returns original utterance with [MASK] replaced with highest scoring word-piece.
"""
logits = logits.detach().cpu().numpy()
token_ids = token_ids.detach().cpu().numpy()
res = []
num_sequences = len(logits)
assert num_sequences == len(token_ids)
for seq_id in range(num_sequences):
# get predicted wp
wp_id = np.where(token_ids[seq_id] == mask_token_id)
assert len(wp_id) == 1
logits_for_masked_wp = logits[seq_id][
wp_id] # shape is now [vocab_size]
tag_wp_id = np.asscalar(np.argmax(logits_for_masked_wp))
tag_wp = id2mlm_tag[tag_wp_id]
# fill in input sequence
mlm_in = utterances[seq_id]
filled_in_sequence = mlm_in.copy()
filled_in_sequence[mlm_in.index('[MASK]')] = tag_wp
res.append(filled_in_sequence)
return res # sequence with predicted word-piece, one per sequence in batch
def lr_one_pytorch_hjmshi_lbfgs(
x: torch.tensor,
y: torch.tensor,
history_size=10,
max_iter=100,
max_ls=25,
tol=1e-4,
C=1,
):
print("XXX: This seems to be broken currently.")
# XXX: Currently broken
from .vendor.hjmshi_lbfgs import FullBatchLBFGS
model = RegLogitModel(x.shape[-1], 1, C=C).to(x.device)
optimizer = FullBatchLBFGS(
model.parameters(),
lr=1,
history_size=history_size,
line_search="Wolfe",
)
x_var = x.detach()
x_var.requires_grad_(True)
y_var = y.detach().float()
def closure():
loss = model.forward_loss(x_var, y_var)
return loss
loss = closure()
loss.backward()
n_iter = 1
while 1:
options = {
"closure": closure,
"current_loss": loss,
"max_ls": max_ls,
}
(
loss,
grad,
lr,
backtracks,
clos_evals,
grad_evals,
desc_dir,
fail,
) = optimizer.step(options=options)
grad_max = grad.abs().max()
if fail:
raise RuntimeError(
"Optimizer failure in lr_one_pytorch_hjmshi_lbfgs", fail)
elif torch.isnan(loss):
raise RuntimeError("NaN loss in lr_one_pytorch_hjmshi_lbfgs")
elif grad_max < tol or n_iter == max_iter:
return (model.linear.weight.detach(), model.linear.bias.detach(),
n_iter)
n_iter += 1
def push(self, samples: torch.tensor, class_ids: torch.tensor) -> NoReturn:
samples = samples.detach().to("cpu")
class_ids = class_ids.detach().to("cpu")
for sample, class_id in zip(samples, class_ids):
self.buffer.append((sample, class_id))
if len(self.buffer) > self.max_samples:
self.buffer.pop(0)
def write_step(self, logits_flat: torch.tensor, targets_flat: torch.tensor,
step: int, ignore_index: int):
logits_flat_np = logits_flat.detach().cpu().numpy()
targets_flat_np = targets_flat.detach().cpu().numpy()
tmp1 = np.argmax(logits_flat_np, axis=1) == targets_flat_np
tmp2 = tmp1[targets_flat_np != ignore_index]
acc = np.sum(tmp2) / np.sum(targets_flat_np != ignore_index)
self.writer.add_scalar('Acc/top-1 step', acc, step)
self.accuracies.append(acc)
def _store_run_stats(self, agent_likelihood: torch.tensor,
prior_likelihood: torch.tensor,
augmented_likelihood: torch.tensor, score):
self._run_stats.append({
"agent_likelihood":
agent_likelihood.detach().mean(),
"prior_likelihood":
prior_likelihood.detach().mean(),
"augmented_likelihood":
augmented_likelihood.detach().mean(),
"score":
np.mean(score)
})
def __call__(
self,
score: torch.tensor,
pred: torch.tensor,
gt: torch.tensor,
pred_class: torch.tensor = False) -> Tuple[
torch.tensor, torch.tensor, torch.tensor]:
"""calculate the sorted overlap matrix between prediction and ground
truth.
remaining_shape is defined by the iou_calculator function
Args:
score (torch.tensor): (N_{predictions})
the score of the predictions
pred (torch.tensor): (N_{predictions}, *remaining_shape)
the prediction itself. Type of prediction depends on
iou_calculator.
gt (torch.tensor): (N_{ground_truth}, *remaining_shape)
the ground truth data to compare the predictions with
pred_class (torch.LongTensor, optional): (N_{predictions})
defaults to None. predictions of the classes if there are any.
Returns:
score (torch.tensor): (N_{filtered_predictions})
the filtered and sorted scores
iou (torch.tensor): (N_{filtered_predictions}, N_{ground_truth})
the iou matrix which compares predictions to its ground truth.
pred_class (Union[torch.LongTensor, None):
(N_{filtered_predictions}) returns the predicted class of the
filtered and sorted predictions if given
"""
score = score.detach()
pred = pred.detach()
if self.score_threshold is not None:
valid_indicator = score >= self.score_threshold
score = score[valid_indicator]
pred = pred[valid_indicator]
if pred_class is not None:
pred_class = pred_class[valid_indicator]
if self.sort:
score, indices = score.sort(descending=True)
pred = pred[indices]
if pred_class is not None:
pred_class = pred_class[indices]
iou = self.iou_calculator(pred, gt)
return score, iou, pred_class
def gap(predictions: torch.tensor, confidences: torch.tensor,
targets: torch.tensor) -> float:
"""Global Average Precision.
Args:
predictions (torch.tensor): predicted class,
should have a size (batch,)
confidences (torch.tensor): predicted confidence (value),
should have a size (batch,)
targets (torch.tensor): targets,
should have a size (batch,)
Returns:
float: [description]
"""
assert len(predictions.shape) == 1
assert len(confidences.shape) == 1
assert len(targets.shape) == 1
assert predictions.shape == confidences.shape == targets.shape
_, indices = torch.sort(confidences, descending=True)
confidences = confidences.detach().cpu().numpy()
predictions = predictions[indices].detach().cpu().numpy()
targets = targets[indices].detach().cpu().numpy()
accum, true_pos = 0.0, 0.0
for i, (conf, pred,
tgt) in enumerate(zip(confidences, predictions, targets)):
match = int(pred == tgt)
true_pos += match
accum += true_pos / (i + 1) * match
accum /= targets.shape[0]
return accum
def to_numpy(gpu_tensor: torch.tensor) -> np.ndarray:
"""
Make numpy array from cuda tensor
:param gpu_tensor: cuda tensor
:return: numpy array
"""
return gpu_tensor.detach().cpu().numpy()
def show_pic(self, t: torch.tensor):
t = t.detach()
# print('------------所有数据-------------')
# print(t.mean(),t.std())
# print(t.shape)
# print('------------固定batchSize-------------')
x = torch.tensor(
[t[i, :, :].std() for i in range(min(t.shape[0], 20))])
# print(x)
# print(x.mean(),x.std())
a = x.std()
aa = x.mean()
# print('------------固定sentenseSize-------------')
x = torch.tensor(
[t[:, i, :].std() for i in range(min(t.shape[1], 20))])
# print(x)
# print(x.mean(),x.std())
b = x.std()
bb = x.mean()
# print('------------固定hiddenSize-------------')
x = torch.tensor(
[t[:, :, 64 * i:64 * (i + 1)].std() for i in range(8)])
# print(x)
# print(x.mean(),x.std())
c = x.std()
cc = x.mean()
return a, b, c, aa, bb, cc
def _forward(self, pixel_model, pixel_x: torch.tensor, target,
avoid_target: bool, scale_eps: bool):
# if scale_eps is True, change eps adaptively. this usually improve robustness against wide range of attack
if scale_eps:
if self.scale_each:
rand = torch.rand(pixel_x.size()[0], device=self.device)
else:
rand = random.random() * torch.ones(pixel_x.size()[0],
device=self.device)
base_eps = rand.mul(self.eps_max)
step_size = rand.mul(self.step_size)
else:
base_eps = self.eps_max * torch.ones(pixel_x.size(0),
device=self.device)
step_size = self.step_size * torch.ones(pixel_x.size(0),
device=self.device)
# init delta
pixel_input = pixel_x.detach()
pixel_input.requires_grad_()
pixel_delta = self._init_delta(pixel_input.size(), base_eps)
# compute delta in pixel space
if self.num_iteration: # run iteration
pixel_delta = self._run(pixel_model, pixel_input, pixel_delta,
target, avoid_target, base_eps, step_size)
else: # if self.num_iteration is 0, return just initialization result
pixel_delta.data = torch.clamp(pixel_input.data, +pixel_delta.data,
0.0, 255.0) - pixel_input.data
# IMPORTANT: this return is in PIXEL SPACE (=[0,255])
return pixel_input + pixel_delta
def lr_one_skl(x: torch.tensor, y: torch.tensor, **kwargs):
import warnings
from sklearn.exceptions import ConvergenceWarning
from sklearn.linear_model import LogisticRegression
initial_device = x.device
x = x.detach()
model = LogisticRegression(**kwargs)
with warnings.catch_warnings():
warnings.simplefilter("ignore", ConvergenceWarning)
try:
model = model.fit(x, y)
except ValueError as exc:
if exc.args[0].startswith(
"This solver needs samples of at least 2 classes"):
return None
raise exc
weight = model.coef_
bias = model.intercept_
weight = torch.as_tensor(weight,
dtype=torch.float32,
device=initial_device)
bias = torch.as_tensor(bias, dtype=torch.float32, device=initial_device)
n_iter = model.n_iter_
return weight, bias, n_iter
def write_step(self, logits_flat: torch.tensor, targets_flat: torch.tensor,
step: int, ignore_index: int):
logits_flat_np = logits_flat.detach().cpu().numpy()
targets_flat_np = targets_flat.detach().cpu().numpy()
tmp1 = np.argmax(logits_flat_np, axis=1) == targets_flat_np
tmp2 = tmp1[targets_flat_np != ignore_index]
acc = np.sum(tmp2) / np.sum(targets_flat_np != ignore_index)
# acc = np.sum(np.argmax(logits_flat_np, axis=1) == targets_flat_np) / targets_flat_np.size
# acc10 = np.sum(
# logits_flat_np.argsort()[:, -10:] == np.expand_dims(targets_flat_np, axis=1)) / targets_flat_np.size
self.writer.add_scalar('Acc/top-1 step', acc, step)
# self.writer.add_scalar('Acc/top-10 step', acc10, step)
self.accuracies.append(acc)
def forward(self, logits: torch.tensor, keywords: torch.tensor):
"""calculate keyword loss
Args:
logits (torch.tensor): B, S2, V
keywords (torch.tensor): B, S1
"""
# logits = nn.functional.gelu(logits)
logits = logits.mean(1) # B, V
logits[:, 0] = 0
logits = torch.log_softmax(logits, -1)
mask = (keywords == 101) | (keywords == 102) | (keywords == 117) | (
keywords == 120) | (keywords == 0)
kws = keywords.detach().clone()
kws[mask] = 0
# B, vocab_size
kw_matrix = torch.zeros(
(logits.shape[0], logits.shape[-1])).to(logits.device)
for i, row in enumerate(kws):
kw_matrix[i, :] = 0.
kw_matrix[i, row] = self.alpha
kw_matrix[i, 0] = 0.
# print(kw_matrix)
kw_matrix = torch.softmax(kw_matrix, -1)
# kw_matrix.requires_grad = False
# print(kw_matrix)
return self.criterion(logits, kw_matrix)
def update_boxes(self, bboxes: torch.tensor, conv2: torch.tensor):
"""
Refine bounding boxes proposed by BBoxRegressor
:tensor bboxes: bboxes from conv5 (19x25x9x4); 4 -> H, W, x, y
:tensor conv2: conv2 feature cube (64x8x150x200)
"""
pdb.set_trace()
scaled_bboxes = bboxes.detach().clone()
scaled_bboxes[:, [0, 3]] *= 200 / 19 # update H, y
scaled_bboxes[:, [1, 2]] *= 150 / 25 # update W, x
scaled_bboxes = scaled_bboxes.numpy().astype(np.int64)
# slice tubes from conv2 feature map
tubes = conv2.data[:, :, :, scaled_bboxes[:, 2] -
scaled_bboxes[:, 1] / 2:scaled_bboxes[:, 2] +
scaled_bboxes[:, 1] / 2, scaled_bboxes[:, 3] -
scaled_bboxes[:, 0] / 2:scaled_bboxes[:, 3] +
scaled_bboxes[:, 0] / 2]
x1 = self.toi2(tubes)
x1 = torch.norm(x1, p=2) # Cx8x8x8
x2 = self.toi5(bboxes)
x2 = x2.repeat(1, 8, 1, 1, 1) # Cx8x4x4
x2 = torch.norm(x2, p=2)
x = torch.cat((x1, x2), dim=1) # Cx8x12x12
x = x.reshape((512, 8, -1)) # CxDx144
x = self.conv11(x)
reg = torch.flatten(x)
reg = self.fc6(reg)
reg = self.fc7(reg)
reg = self.fc8(reg)
return reg
def get_dense_matrix(self, data: torch.tensor, c: torch.tensor):
batch_size = c[-1, -1] + 1
data = spconv.SparseConvTensor(data.unsqueeze(1),
c[:, self.permute_tensor],
self.spatial_size, batch_size)
data = data.dense()
data = data.detach().cpu().numpy()
return data
def plot_flow(flow: torch.tensor):
figure = plt.figure(figsize=(10, 5))
np_flow = flow.detach().cpu().numpy().transpose(1, 2, 0)
np_flow = flow_to_color(np_flow)
plt.axis("off")
plt.imshow(np_flow)
plot = np.asarray(fig2img(figure, height=HEIGHT, width=WIDTH))
plt.close(figure)
return plot
def plot_disp(disp: torch.tensor):
figure = plt.figure(figsize=(10, 5))
np_disp = disp.detach().cpu().squeeze(0).numpy()
vmax = np.percentile(np_disp, 95)
plt.imshow(np_disp, cmap="magma", vmax=vmax)
plt.axis("off")
plot = np.asarray(fig2img(figure, height=HEIGHT, width=WIDTH))
plt.close(figure)
return plot
def plot_depth(depth: torch.tensor):
figure = plt.figure(figsize=(10, 5))
np_depth = depth.detach().cpu().squeeze(0).numpy()
vmax = np.percentile(np_depth, 95)
plt.imshow(np_depth, cmap="gray_r")
plt.axis("off")
plot = np.asarray(fig2img(figure, height=HEIGHT, width=WIDTH))
plt.close(figure)
return plot
def show_sec(self, t: torch.tensor):
def f(x):
return format(float(x), '.3f')
t = t.detach()
for i in range(min(t.shape[0], 20)):
a, b, c = t[i, :, :].std(), t[i, :, :].mean(), self._get_symm_kl(
t[i, :, :])
print(f(a), ' ', f(b))
def resize(img:torch.tensor):
img_np = img.round().byte().detach().cpu().numpy()
img_np = cv2.resize(img_np, (800,800), interpolation=cv2.INTER_LINEAR)
img_np = torch.from_numpy(img_np).float().to(device)
img = img.permute(2,0,1)
img = nn.functional.interpolate(img.unsqueeze(0), size=(800, 800),mode="bilinear", align_corners=False)
img = img.squeeze(0).permute(1,2,0)
img = img + (img_np-img.detach())
return img
def _clip_loss(
self,
action_logprobs: torch.tensor,
new_logprobs: torch.tensor,
advantages: torch.tensor,
):
ratio = torch.exp(new_logprobs - action_logprobs.detach())
clipped_ratio = torch.clamp(ratio, 1 - self.ppo_epsilon, 1 + self.ppo_epsilon)
# minus sign is needed to do gradient ascent
actor_loss = -(torch.min(ratio * advantages, clipped_ratio * advantages)).mean()
return actor_loss
def lr_one_nlesc_dirac_lbgfs(
x: torch.tensor,
y: torch.tensor,
history_size=10,
max_iter=100,
max_ls=25,
tol=1e-3,
C=1,
):
from .vendor.nlesc_dirac_lbgfs import LBFGSNew
model = RegLogitModel(x.shape[-1], 1, C=C).to(x.device)
optimizer = LBFGSNew(
model.parameters(),
lr=1,
max_iter=max_iter,
history_size=history_size,
tolerance_grad=tol,
tolerance_change=0,
line_search_fn=True,
batch_mode=False,
)
x_var = x.detach()
x_var.requires_grad_(True)
y_var = y.detach().float()
def closure():
if torch.is_grad_enabled():
optimizer.zero_grad()
loss = model.forward_loss(x_var, y_var)
if torch.is_grad_enabled():
loss.backward()
return loss
optimizer.step(closure)
state = optimizer.state[next(iter(optimizer.state))]
return (model.linear.weight.detach(), model.linear.bias.detach(),
state["n_iter"])
def get_accuracy(model_prediction: torch.tensor,
true_prediction: torch.tensor) -> float:
"""
Calculates accuracy with model predictions
Model predictions are the raw model output
True predictions are the labels
:param model_prediction: raw model output
:param true_prediction: labels
:return: accuracy
"""
model_prediction: np.ndarray = model_prediction.detach().cpu().numpy()
true_prediction: np.ndarray = true_prediction.detach().cpu().numpy()
model_prediction = model_prediction.argmax(axis=1)
accuracy_number = np.sum(
model_prediction == true_prediction) / len(true_prediction)
return accuracy_number
def calc_gradient_penalty(params: utils.Params, d_model: cgan.Discriminator,
obs_mag: torch.tensor, g_out: torch.tensor,
conditions: torch.tensor,
true_mag: torch.tensor) -> torch.tensor:
"""Calculate the gradient penalty
Args:
params (utils.Params): the hyperparameters used for training
d_model (cgan.Discriminator): the Discriminator model
obs_mag (torch.tensor): the ground truth observed galaxy magnitudes
g_out (torch.tensor): the output of the Generator at this step
conditions (torch.tensor): the observing conditions used as inputs
true_mag (torch.tensor): the ground truth true galaxy magnitudes
Returns:
(torch.tensor): gradient penalty
"""
alpha = torch.rand(conditions.shape[0], 1)
alpha = alpha.expand(conditions.shape[0], 3).contiguous()
if params.cuda:
alpha = alpha.cuda(non_blocking=True)
interpolates = alpha * obs_mag.detach() + ((1 - alpha) * g_out.detach())
interpolates.requires_grad_(True)
disc_interpolates = d_model(interpolates, conditions, true_mag)
ones = torch.ones(disc_interpolates.size())
if params.cuda:
ones = ones.cuda(non_blocking=True)
gradients = grad(outputs=disc_interpolates, inputs=interpolates,
grad_outputs=ones, create_graph=True, retain_graph=True,
only_inputs=True)[0]
gradients = gradients.view(gradients.size(0), -1)
gradient_penalty = ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10
return gradient_penalty
def lr_one_pytorch_lbfgs(
x: torch.tensor,
y: torch.tensor,
history_size=10,
max_iter=100,
max_ls=25,
tol=1e-4,
C=1,
):
from torch.optim import LBFGS
model = RegLogitModel(x.shape[-1], 1, C=C).to(x.device)
optimizer = LBFGS(
model.parameters(),
lr=1,
history_size=history_size,
max_iter=max_iter,
# XXX: Cannot pass max_ls to strong_wolfe
line_search_fn="strong_wolfe",
tolerance_change=0,
tolerance_grad=tol,
)
x_var = x.detach()
x_var.requires_grad_(True)
y_var = y.detach().float()
def closure():
if torch.is_grad_enabled():
optimizer.zero_grad()
loss = model.forward_loss(x_var, y_var)
if torch.is_grad_enabled():
loss.backward()
return loss
optimizer.step(closure)
state = optimizer.state[next(iter(optimizer.state))]
return (model.linear.weight.detach(), model.linear.bias.detach(),
state["n_iter"])
def lr_one_cuml(x: torch.tensor, y: torch.tensor, **kwargs):
from cuml.linear_model import LogisticRegression
initial_device = x.device
x = x.detach()
y = y.float()
model = LogisticRegression(**kwargs)
model = model.fit(x, y)
weight = model.coef_
bias = model.intercept_
weight = torch.as_tensor(weight, device=initial_device).float().t()
bias = torch.as_tensor(bias, device=initial_device).float()
n_iter = model.solver_model.num_iters
return weight, bias, n_iter
def glimpse(self, image: torch.tensor,
location: torch.tensor) -> torch.tensor:
image = transforms.ToPILImage()(image)
location = tuple(location.detach().numpy())
coords = self.get_coords(image.size, location)
crops = []
for coord in coords:
crop = image.crop(coord)
crop = crop.resize((self.patch_width, self.patch_width))
crops.append(crop)
crops = [transforms.ToTensor()(x) for x in crops]
crops = torch.stack(crops, dim=0)
return crops
def get_f1_score(model_prediction: torch.tensor,
true_prediction: torch.tensor) -> float:
"""
Calculates weighted f1_score with model predictions
Model predictions are the raw model output
True predictions are the labels
:param model_prediction: raw model output
:param true_prediction: labels
:return: f1 score number
"""
model_prediction: np.ndarray = model_prediction.detach().cpu().numpy()
true_prediction: np.ndarray = true_prediction.detach().cpu().numpy()
model_prediction = model_prediction.argmax(axis=1)
f1_score_number = f1_score(y_true=true_prediction,
y_pred=model_prediction,
average='weighted')
return f1_score_number