def layerwise_relevance_zb(self, out, lo=-1, hi=1, use_bias=False, **kwargs):
if self._in is None:
raise RuntimeError('Block has not yet executed forward_logged!')
R = out
a = self._in[0]
weight = self.weight.data(ctx=a.context)
wplus = nd.maximum(0., weight)
wminus = nd.minimum(0., weight)
bias = None
bplus = None
bminus = None
if use_bias is not None:
bias = self.bias.data(ctx=a.context)
bplus = nd.maximum(0., bias)
bminus = nd.minimum(0., bias)
upper = nd.ones_like(a)*hi
lower = nd.ones_like(a)*lo
a.attach_grad()
upper.attach_grad()
lower.attach_grad()
with autograd.record():
zlh = ( self._forward(a, weight, bias)
- self._forward(lower, wplus, bplus)
- self._forward(upper, wminus, bminus)
)
zlh.backward(out_grad=R/(zlh + (zlh == 0.)))
return a*a.grad + upper*upper.grad + lower*lower.grad
def layerwise_relevance_zb(self,
out,
lo=-1,
hi=1,
use_bias=False,
**kwargs):
if self._in is None:
raise RuntimeError('Block has not yet executed forward_logged!')
R = out
a = self._in[0]
weight = self.weight.data(ctx=a.context)
wplus = nd.maximum(0., weight)
wminus = nd.minimum(0., weight)
bias = None
bplus = None
bminus = None
if use_bias is not None:
bias = self.bias.data(ctx=a.context)
bplus = nd.maximum(0., bias)
bminus = nd.minimum(0., bias)
upper = nd.ones_like(a) * hi
lower = nd.ones_like(a) * lo
a.attach_grad()
upper.attach_grad()
lower.attach_grad()
with autograd.record():
zlh = (self._forward(a, weight, bias) -
self._forward(lower, wplus, bplus) -
self._forward(upper, wminus, bminus))
zlh.backward(out_grad=R / (zlh + (zlh == 0.)))
return a * a.grad + upper * upper.grad + lower * lower.grad
def _ohem_single(self, score_gt, score_pred, training_masks):
if self.debug:
print("score_gt_shape:", score_gt.shape, "score_pred_shape:", score_pred.shape, \
"train_mask_shape:", training_masks.shape)
pos_gt_thres = F.where(score_gt > 0.5, F.ones_like(score_gt), F.zeros_like(score_gt))
pos_num = F.sum(pos_gt_thres) - F.sum(pos_gt_thres * training_masks)
if pos_num == 0:
selected_mask = training_masks
return selected_mask
neg_lt_thres = F.where(score_gt <= 0.5, F.ones_like(score_gt), F.zeros_like(score_gt))
neg_num = F.sum(neg_lt_thres)
neg_num = min(pos_num * 3, neg_num)
if neg_num == 0:
selected_mask = training_masks
return training_masks
neg_score = neg_lt_thres * score_pred
neg_score_sorted = F.sort(neg_score.reshape(-1), is_ascend=0, axis=None)
threshold = neg_score_sorted[neg_num - 1]
score_gt_thres = F.where(score_pred >= threshold, F.ones_like(score_pred), F.zeros_like(score_pred))
trained_sample_mask = F.logical_or(score_gt_thres, pos_gt_thres)
selected_mask = F.logical_and(trained_sample_mask, training_masks)
return selected_mask
def forward(self, pred, label, valid_length): # pylint: disable=arguments-differ
"""
Parameters
----------
F
pred : Symbol or NDArray
Shape (batch_size, length, V)
label : Symbol or NDArray
Shape (batch_size, length)
valid_length : Symbol or NDArray
Shape (batch_size, )
Returns
-------
loss : Symbol or NDArray
Shape (batch_size,)
"""
if self._sparse_label:
sample_weight = nd.cast(nd.expand_dims(nd.ones_like(label), axis=-1), dtype=np.float32)
else:
sample_weight = nd.ones_like(label)
sample_weight = nd.SequenceMask(sample_weight,
sequence_length=valid_length,
use_sequence_length=True,
axis=1)
return super(SoftmaxCEMaskedLoss, self).forward( pred, label, sample_weight)
def sample(match, cls_pred, iou, ratio=3, min_sample=0, threshold=0.5, do=True):
if do is False:
ones = nd.ones_like(match)
sample = nd.where(match > -0.5, ones, ones*-1)
return sample
sample = nd.zeros_like(match)
num_pos = nd.sum(match > -0.5, axis=-1)
requre_neg = ratio * num_pos
neg_mask = nd.where(match < -0.5, nd.max(iou, axis=-1) < threshold, sample)
max_neg = neg_mask.sum(axis=-1)
num_neg = nd.minimum(max_neg, nd.maximum(requre_neg, min_sample)).astype('int')
neg_prob = cls_pred[:,:,0]
max_value = nd.max(cls_pred, axis=-1, keepdims=True)
score = max_value[:,:,0] - neg_prob + nd.log(
nd.sum(
nd.exp(cls_pred-max_value), axis=-1))
score = nd.where(neg_mask, score, nd.zeros_like(score))
argmax = nd.argsort(score, axis=-1, is_ascend=False)
sample = nd.where(match > -0.5, nd.ones_like(sample), sample)
for i, num in enumerate(num_neg):
sample[i, argmax[i,:num.asscalar()]] = -1
return sample
def get_accuracy(pre_l, true_l):
one_zero_pre = nd.where(pre_l > 0.5, nd.ones_like(pre_l),
nd.zeros_like(pre_l))
compare = nd.equal(one_zero_pre, true_l).sum(axis=1)
samples_right = nd.where(compare == 3, nd.ones_like(compare),
nd.zeros_like(compare)).sum()
all_num = pre_l.shape[0]
return samples_right / all_num
def main():
x = nd.arange(20)
A = x.reshape(shape=(5, 4))
print(A)
print('A[2, 3] = ', A[2, 3])
print('A[2, :] = ', A[2, :])
print('A[:, 3] = ', A[:, 3])
print('A.T = ', A.T)
X = nd.arange(24).reshape(shape=(2, 3, 4))
print(X)
u = nd.array([1, 2, 4, 8])
v = nd.ones_like(u) * 2
print('u + v = ', u + v)
print('u - v = ', u - v)
print('u * v = ', u * v)
print('u / v = ', u / v)
B = nd.ones_like(A) * 3
print('B = ', B)
print('A + B = ', A + B)
print('A * B = ', A * B)
a = 2
x = nd.ones(3)
y = nd.zeros(3)
print(x.shape)
print(y.shape)
print((a * x).shape)
print((a * x + y).shape)
print(nd.sum(u))
print(nd.sum(A))
print(nd.mean(A))
print(nd.sum(A) / A.size)
print(nd.dot(u, v))
print(nd.sum(u * v))
print(nd.dot(A, u))
A = nd.ones(shape=(3, 4))
B = nd.ones(shape=(4, 5))
print(nd.dot(A, B))
# L2 norm
print(nd.norm(u))
# L1 norm
print(nd.sum(nd.abs(u)))
def match(iou, threshould=0.5, share_max=False):
B, N, M = iou.shape
if share_max:
result = nd.argmax(iou, axis=-1)
result = nd.where(nd.max(iou, axis=-1) > threshould, result, nd.ones_like(result)*-1)
else:
match = [getUniqueMatch(i) for i in iou]
result = nd.concat(*match, dim=0)
argmax_row = nd.argmax(iou, axis=-1)
max_row = nd.max(iou, axis=-1)
argmax_row = nd.where(max_row > threshould, argmax_row, nd.ones_like(argmax_row)*-1)
result = nd.where(result > -0.5, result, argmax_row)
return result
def get_cls_targets(cls_targets_1, cls_targets_2):
cls_targets = []
for (cls_target_1, cls_target_2) in zip(cls_targets_1, cls_targets_2):
cls_target_1_idx = nd.where(cls_target_1 > 0,
nd.ones_like(cls_target_1),
nd.zeros_like(cls_target_1))
cls_target_2_idx = nd.where(cls_target_2 > 0,
nd.ones_like(cls_target_2),
nd.zeros_like(cls_target_2))
cls_target_idx = nd.where(cls_target_1_idx == cls_target_2_idx, nd.ones_like(cls_target_1_idx),\
nd.zeros_like(cls_target_1_idx))
cls_target = nd.where(cls_target_idx, cls_target_1,
nd.ones_like(cls_target_1) * -1)
cls_targets.append(cls_target)
return cls_targets
def forward(self, bboxes, anchors, height, width):
# 标注ious
with autograd.pause():
ious = mx.nd.contrib.box_iou(anchors, bboxes)
# 去除无效的锚框(超出边界的)
x_min, y_min, x_max, y_max = self._spliter(anchors)
invalid_mask = (x_min < 0) + (y_min < 0) + (x_max >= width) + (
y_max >= height)
# 将所有无效锚框的ious设为-1
invalid_mask = nd.repeat(invalid_mask,
repeats=bboxes.shape[0],
axis=-1)
ious = nd.where(invalid_mask > 0, nd.ones_like(ious) * -1, ious)
# 对锚框进行采样
samples, matches = self._sampler(ious)
# 下面进行标注
cls_label, _ = self._cls_encoder(samples)
targets, masks = self._bbox_encoder(samples.expand_dims(axis=0),
matches.expand_dims(axis=0),
anchors.expand_dims(axis=0),
bboxes.expand_dims(axis=0))
return cls_label, targets[0], masks[0]
def volume_render_radiance_field(radiance_field, depth_values, ray_directions, radiance_field_noise_std=0.0,
white_background=False):
# TESTED
one_e_10 = nd.array([1e10], dtype=ray_directions.dtype, ctx=ray_directions.context).broadcast_to(depth_values[..., :1].shape)
dists = nd.concat(*[depth_values[..., 1:] - depth_values[..., :-1], one_e_10], dim=-1)
dists = dists * ray_directions[..., None, :].norm(ord=2, axis=-1)
rgb = nd.sigmoid(radiance_field[..., :3])
noise = 0.0
if radiance_field_noise_std > 0.0:
noise = nd.random.normal(0.0, 1.0, shape=radiance_field[..., 3].shape,
dtype=radiance_field.dtype, ctx=radiance_field.context)
noise = noise * radiance_field_noise_std
sigma_a = nd.relu(radiance_field[..., 3] + noise)
alpha = 1.0 - nd.exp(-sigma_a * dists)
weights = alpha * cumprod_exclusive_gluon(1.0 - alpha + 1e-10)
rgb_map = weights[..., None] * rgb
rgb_map = rgb_map.sum(axis=-2)
depth_map = weights * depth_values
depth_map = depth_map.sum(axis=-1)
# depth_map = (weights * depth_values).sum(dim=-1)
acc_map = weights.sum(axis=-1)
disp_map = 1.0 / nd.maximum(1e-10 * nd.ones_like(depth_map), depth_map / acc_map)
if white_background:
rgb_map = rgb_map + (1.0 - acc_map[..., None])
return rgb_map, disp_map, acc_map, weights, depth_map
def get_xps(weight_denominator, weight_numerator, z):
xps = list()
xps.append(z)
for _ in range(max(len(weight_numerator), len(weight_denominator))):
xps.append(nd.multiply(xps[-1], z))
xps.insert(0, nd.ones_like(z))
return xps
def hybrid_forward(self, F, samples, matches, refs):
"""HybridBlock, handle multi batch correctly
Parameters
----------
samples: (B, N), value +1 (positive), -1 (negative), 0 (ignore)
matches: (B, N), value range [0, M)
refs: (B, M), value range [0, num_fg_class), excluding background
Returns
-------
targets: (B, N), value range [0, num_fg_class + 1), including background
"""
# samples (B, N) (+1, -1, 0: ignore), matches (B, N) [0, M), refs (B, M)
# reshape refs (B, M) -> (B, 1, M) -> (B, N, M)
refs = F.repeat(refs.reshape((0, 1, -1)),
axis=1,
repeats=matches.shape[1])
# ids (B, N, M) -> (B, N), value [0, M + 1), 0 reserved for background class
target_ids = F.pick(refs, matches, axis=2) + 1
# samples 0: set ignore samples to ignore_label
targets = F.where(samples > 0.5, target_ids,
nd.ones_like(target_ids) * self._ignore_label)
# samples -1: set negative samples to 0
targets = F.where(samples < -0.5, nd.zeros_like(targets), targets)
return targets
def condition(self, y):
z = nd.ones_like(y)
if self._ymin is not None:
z = z * (y > self._ymin)
if self._ymax is not None:
z = z * (y < self._ymax)
return z
def hybrid_forward(self, F, score_gt, kernel_gt, score_pred, training_masks, *args, **kwargs):
"""
kernels map's order: [1, ..., 0.5]
"""
C_pred = score_pred[:, 0, :, :]
self.pixel_acc = batch_pix_accuracy(C_pred, score_gt)
# classification loss
eps = 1e-5
intersection = F.sum(score_gt * C_pred * training_masks, axis=(1, 2))
union = F.sum(training_masks * score_gt * score_gt, axis=(1, 2)) + F.sum(training_masks * C_pred * C_pred, axis=(1, 2)) + eps
C_dice_loss = 1. - (2 * intersection) / (union)
# loss for kernel
kernel_mask = F.where(training_masks * C_pred > 0.5, F.ones_like(C_pred), F.zeros_like(C_pred))
kernel_mask = F.expand_dims(kernel_mask, axis=1)
kernel_mask = F.repeat(kernel_mask, repeats=self.num_kernels-1, axis=1)
self.kernel_acc = batch_pix_accuracy(score_pred[:, 1, :, :] * score_gt, kernel_gt[:, 0, :, :])
kernel_intersection = F.sum(kernel_gt * score_pred[:, 1:, :, :] * kernel_mask, axis=(2, 3))
kernel_union = F.sum(kernel_gt * kernel_gt * kernel_mask, axis=(2, 3)) + F.sum(score_pred[:, 1:, :, :] * score_pred[:, 1:, :, :] * kernel_mask, axis=(2, 3)) + eps
kernel_dice = 1. - (2 * kernel_intersection) / kernel_union
kernel_dice_loss = F.mean(kernel_dice, axis=1)
self.C_loss = C_dice_loss
self.kernel_loss = kernel_dice_loss
loss = self.lam * C_dice_loss + (1. - self.lam) * kernel_dice_loss
return loss
def label_offset(anchors, bbox, match, sample,
means=(0,0,0,0), stds=(0.1,0.1,0.2,0.2), flatten=True):
anchors = anchors.reshape((-1,4))
N, _ = anchors.shape
B, M, _ = bbox.shape
anchor_x, anchor_y, anchor_w, anchor_h = corner_to_center(anchors, split=True)
bbox = bbox.reshape((B,1,M,4))
bbox = nd.broadcast_to(bbox, (B,N,M,4))
bbox = nd.stack(*[nd.pick(bbox[:,:,:,p], match) for p in range(4)], axis=-1)
bbox_x, bbox_y, bbox_w, bbox_h = corner_to_center(bbox, split=True)
offset_x = ((bbox_x - anchor_x) / anchor_w - means[0]) / stds[0]
offset_y = ((bbox_y - anchor_y) / anchor_h - means[1]) / stds[1]
offset_w = (nd.log(bbox_w/anchor_w) - means[2]) / stds[2]
offset_h = (nd.log(bbox_h/anchor_h) - means[3]) / stds[3]
offset = nd.concat(*(offset_x, offset_y, offset_w, offset_h), dim=-1)
sample = sample.reshape((B,N,1))
sample = nd.broadcast_to(sample, (B,N,4)) > 0.5
anchor_offset = nd.where(sample, offset, nd.zeros_like(offset))
anchor_mask = nd.where(sample, nd.ones_like(offset), nd.zeros_like(offset))
if flatten:
anchor_offset = anchor_offset.reshape((B,-1))
anchor_mask = anchor_mask.reshape((B,-1))
return anchor_mask, anchor_offset
def accuracy(predictions, targets):
# predictions = nd.argmax(predictions, 1)
# targets = nd.argmax(targets, 1)
# return nd.mean(nd.equal(predictions, targets)).asscalar() * 100
predictions = nd.where(predictions > 0.5, nd.ones_like(predictions),
nd.zeros_like(predictions))
return 100 - nd.mean(nd.abs(predictions - targets)).asscalar() * 100
def test_periodic_kernel(x1, x2, amplitude, length_scale, exact) -> None:
tol = 1e-5
batch_size = amplitude.shape[0]
history_length_1 = x1.shape[0]
history_length_2 = x2.shape[0]
num_features = x1.shape[1]
if batch_size > 1:
x1 = nd.tile(x1, reps=(batch_size, 1, 1))
x2 = nd.tile(x2, reps=(batch_size, 1, 1))
for i in range(1, batch_size):
x1[i, :, :] = (i + 1) * x1[i, :, :]
x2[i, :, :] = (i - 3) * x2[i, :, :]
else:
x1 = x1.reshape(batch_size, history_length_1, num_features)
x2 = x2.reshape(batch_size, history_length_2, num_features)
amplitude = amplitude.reshape(batch_size, 1, 1)
length_scale = length_scale.reshape(batch_size, 1, 1)
frequency = 1 / 24 * nd.ones_like(length_scale)
periodic = PeriodicKernel(amplitude, length_scale, frequency)
exact = amplitude * nd.exp(
-2 * nd.sin(frequency * math.pi * nd.sqrt(exact))**2 / length_scale**2)
res = periodic.kernel_matrix(x1, x2)
assert nd.norm(exact - res) < tol
def run_epoch(epoch, data_iter, model, trainer, loss_fn, ctx = mx.cpu()):
"Standard Training and Logging Function"
start = time.time()
total_tokens = 0
tokens = 0
total_loss = 0
for i, batch in enumerate(data_iter):
src = batch.src.as_in_context(ctx)
trg = batch.trg.as_in_context(ctx)
src_mask = batch.src_mask.as_in_context(ctx)
trg_mask = batch.trg_mask.as_in_context(ctx)
trg_y = batch.trg_y.as_in_context(ctx)
ntokens = batch.ntokens
with autograd.record():
out = model(src, trg, src_mask, trg_mask)
_out = out.reshape(-1, out.shape[-1])
_cols = list(batch.trg_y.reshape(-1).asnumpy())
_rows = list(range(_out.shape[0]))
_idx = nd.array([_rows, _cols], ctx = ctx)
_trg = nd.scatter_nd(nd.ones_like(trg_y.reshape(-1)), _idx, _out.shape)
loss = nd.sum(loss_fn(_out, _trg))
loss.backward()
trainer.step(out.shape[0])
total_loss += loss.asnumpy()[0]
total_tokens += ntokens.asnumpy()[0]
tokens += ntokens.asnumpy()[0]
if i % 50 == 0:
elapsed = time.time() - start
logger.info("Epoch Step: %d Loss: %f Tokens per Sec: %f" % (epoch, loss.asnumpy()[0] / ntokens.asnumpy()[0], tokens / elapsed))
start = time.time()
tokens = 0
return total_loss #/ total_tokens
def hybrid_forward(self, F, score_gt, kernel_gt, score_pred, training_masks, *args, **kwargs):
# cal ohem mask
selected_masks = []
for i in range(score_gt.shape[0]):
# cal for text region
selected_mask = self._ohem_single(score_gt[i:i+1], score_pred[i:i+1], training_masks[i:i+1])
selected_masks.append(selected_mask)
selected_masks = F.concat(*selected_masks, dim=0)
C_pred = score_pred[:, 0, :, :]
self.pixel_acc = batch_pix_accuracy(C_pred, score_gt)
# classification loss
eps = 1e-5
intersection = F.sum(score_gt * C_pred * selected_masks, axis=(1, 2))
union = F.sum(selected_masks * score_gt * score_gt, axis=(1, 2)) + F.sum(selected_masks * C_pred * C_pred, axis=(1, 2)) + eps
C_dice_loss = 1. - (2 * intersection) / (union)
# loss for kernel
kernel_mask = F.where(training_masks * C_pred > 0.5, F.ones_like(C_pred), F.zeros_like(C_pred))
kernel_mask = F.expand_dims(kernel_mask, axis=1)
kernel_mask = F.repeat(kernel_mask, repeats=self.num_kernels-1, axis=1)
self.kernel_acc = batch_pix_accuracy(score_pred[:, 1, :, :] * score_gt, kernel_gt[:, 0, :, :])
kernel_intersection = F.sum(kernel_gt * score_pred[:, 1:, :, :] * kernel_mask, axis=(2, 3))
kernel_union = F.sum(kernel_gt * kernel_gt * kernel_mask, axis=(2, 3)) + F.sum(score_pred[:, 1:, :, :] * score_pred[:, 1:, :, :] * kernel_mask, axis=(2, 3)) + eps
kernel_dice = 1. - (2 * kernel_intersection) / kernel_union
kernel_dice_loss = F.mean(kernel_dice, axis=1)
self.C_loss = C_dice_loss
self.kernel_loss = kernel_dice_loss
loss = self.lam * C_dice_loss + (1. - self.lam) * kernel_dice_loss
return loss
def condition(self, y):
z = nd.ones_like(y)
if self._ymin is not None:
z = z * (y > self._ymin)
if self._ymax is not None:
z = z * (y < self._ymax)
return z
def forward(self, ious):
matches = nd.argmax(ious, axis=-1)
# 每个锚框最高得分
max_iou_pre_anchor = nd.max(ious, axis=-1)
# 将所有锚框都初始化为0，ignore
samples = nd.zeros_like(max_iou_pre_anchor)
# 计算每个ground_truth 的最高iou
max_all_ious = nd.max(ious, axis=0, keepdims=True)
# 标记处mask中最高分值的那一行为1
mask = nd.broadcast_greater(ious + self._eps, max_all_ious)
mask = nd.sum(mask, axis=-1)
# 将最高分数的锚框标记为 1 正类
samples = nd.where(mask, nd.ones_like(samples), samples)
# 下面标记大于 pos_iou_thresh的样本为正例
samples = nd.where(max_iou_pre_anchor > self._pos_iou_thresh,
nd.ones_like(samples), samples)
# 标记小于neg_iou_thresh的样本为负类
tmp = (max_iou_pre_anchor < self._neg_iou_thresh) * (max_iou_pre_anchor
> 0)
samples = nd.where(tmp, nd.ones_like(samples) * -1, samples)
# 将其转换为 numnpy
samples = samples.asnumpy()
# 下面进行采样
# 首先对正样本进行采样
num_pos = int((samples > 0).sum())
if num_pos > self._max_pos:
discard_indices = np.random.choice(np.where((samples > 0))[0],
size=(num_pos - self._max_pos),
replace=False)
samples[discard_indices] = 0 # 将多余部分设置为忽略
num_neg = int((samples < 0).sum())
max_neg = self._num_sample - min(self._max_pos, num_pos)
if num_neg > max_neg:
discard_indices = np.random.choice(np.where((samples < 0))[0],
size=(num_neg - max_neg),
replace=False)
samples[discard_indices] = 0
# 最后将其转化为ndarray
samples = nd.array(samples, ctx=matches.context)
return samples, matches
def run_one_iter_of_nerf(model_coarse, model_fine, data_dict, options, mode="train",
encode_position_fn=None, encode_direction_fn=None):
height = data_dict['height']
width = data_dict['width']
focal_length = data_dict['focal_length']
ray_origins = data_dict['ray_origins']
ray_directions = data_dict['ray_directions']
viewdirs = None
if options.nerf.use_viewdirs:
# Provide ray directions as input
viewdirs = ray_directions
viewdirs = viewdirs / viewdirs.norm(ord=2, axis=-1).reshape((ray_directions.shape[0], 1))
viewdirs = viewdirs.reshape((-1, 3))
# Cache shapes now, for later restoration.
restore_shapes = [ray_directions.shape, ray_directions.shape[:-1], ray_directions.shape[:-1]]
if model_fine:
restore_shapes += restore_shapes
if options.dataset.no_ndc is False:
ro, rd = ndc_rays(height, width, focal_length, 1.0, ray_origins.reshape((-1, 3)), ray_directions.reshape((-1, 3)))
ro = ro.reshape((-1, 3))
rd = rd.reshape((-1, 3))
else:
ro = ray_origins.reshape((-1, 3))
rd = ray_directions.reshape((-1, 3))
near = options.dataset.near * nd.ones_like(rd[..., :1])
far = options.dataset.far * nd.ones_like(rd[..., :1])
rays = nd.concat(*[ro, rd, near, far], dim=-1)
if options.nerf.use_viewdirs:
rays = nd.concat(*[rays, viewdirs], dim=-1)
batches = get_minibatches(rays, chunksize=getattr(options.nerf, mode).chunksize)
pred = [predict_and_render_radiance(batch, model_coarse, model_fine, options, encode_position_fn=encode_position_fn,
encode_direction_fn=encode_direction_fn) for batch in batches]
synthesized_images = list(zip(*pred))
synthesized_images = [nd.concat(*image, dim=0) if image[0] is not None else None
for image in synthesized_images]
if mode == "validation":
synthesized_images = [image.reshape(shape) if image is not None else None
for (image, shape) in zip(synthesized_images, restore_shapes)]
if len(synthesized_images) == 3:
synthesized_images.append(None)
synthesized_images.append(None)
synthesized_images.append(None)
return tuple(synthesized_images)
def _score_weight_LP(self, mask, ctx):
n = self.LP_negative_weight
p = self.LP_positive_weight
ones = nd.ones_like(mask)
score_weight = nd.where(mask > 0, ones*p, ones*n, ctx=ctx)
return score_weight
def _compute_px_clipping_factors(self, batch_params, num_in_batch):
px_norms = self._compute_px_gradient_norms(batch_params, num_in_batch)
l2_rescales = self._hyperparams['l2_clipping_bound'] / (
px_norms + 1e-8) # tiny additive term to prevent div_by_0
one_vs_rescale = nd.stack(nd.ones_like(px_norms, ctx=self._model_ctx),
l2_rescales,
axis=1)
return nd.min(one_vs_rescale, axis=0, exclude=True)
def operations():
x = nd.array([1, 2, 4, 8])
y = nd.ones_like(x) * 2
print('x =', x)
print('y =', y)
print('x + y', x + y)
print('x - y', x - y)
print('x * y', x * y)
print('x / y', x / y)
def hybrid_forward(self, F, samples, matches, refs):
refs = F.repeat(refs.reshape((0, 1, -1)),
axis=1,
repeats=matches.shape[1])
target_ids = F.pick(refs, matches, axis=2) + 1
targets = F.where(samples > 0.5, target_ids,
nd.ones_like(target_ids) * self._ignore_label)
targets = F.where(samples < -0.5, nd.zeros_like(targets), targets)
return targets
def yolo2_target(scores, boxes, labels, anchors, ignore_label=-1, thresh=0.5):
"""Generate training targets given predictions and labels.
网络预测的输出为(32,16,16,2,5)
而label的形式为：labels即ground truth(32,1,5)，其中5包括一个class label:0，以及左上、右下两个corner相对于整张图的坐标
模型回归的目标形式：
注意：这里传入scores只是为了用其shape和context！
"""
b, h, w, n, _ = scores.shape
anchors = np.reshape(np.array(anchors), (-1, 2))
#scores = nd.slice_axis(outputs, begin=1, end=2, axis=-1)
#boxes = nd.slice_axis(outputs, begin=2, end=6, axis=-1)
gt_boxes = nd.slice_axis(labels, begin=1, end=5, axis=-1)
target_score = nd.zeros((b, h, w, n, 1), ctx=scores.context)
target_id = nd.ones_like(target_score, ctx=scores.context) * ignore_label
target_box = nd.zeros((b, h, w, n, 4), ctx=scores.context)
sample_weight = nd.zeros(
(b, h, w, n, 1), ctx=scores.context
) #注意：sample_weight的设置：只有和真实框的IOU最大的bbox sample_weight为1 ！！
for b in range(output.shape[0]): #b为遍历batch_size个batch中的每一个
# find the best match for each ground-truth
label = labels[b].asnumpy()
# 下一句仅仅是为了过滤掉错误的（小于零）的标签
valid_label = label[np.where(label[:, 0] > -0.5)[0], :]
# shuffle because multi gt could possibly match to one anchor, we keep the last match randomly
np.random.shuffle(valid_label)
for l in valid_label:
gx, gy, gw, gh = (l[1] + l[3]) / 2, (
l[2] + l[4]) / 2, l[3] - l[1], l[4] - l[2]
ind_x = int(gx * w) #算出第几行第几列的cell对当前groundtruth box负责
ind_y = int(gy * h)
tx = gx * w - ind_x # 得出groudtruth的中心坐标相对于要负责的grid cell左上角点的偏移，【【该偏移量即模型要回归的目标数值！！！】】
ty = gy * h - ind_y
gw = gw * w #得出groudtruth box 在feature map上的绝对宽度和高度 如 gw=4.23 gh=6.53
gh = gh * h
# find the best match using width and height only, assuming centers are identical
intersect = np.minimum(anchors[:, 0], gw) * np.minimum(
anchors[:, 1], gh) #计算每个(共两个) anchor box与groundtruth bbox的交集面积
ovps = intersect / (
gw * gh + anchors[:, 0] * anchors[:, 1] - intersect
) # 计算每个(共两个) anchor box与groundtruth bbox的交并比
best_match = int(
np.argmax(ovps)) #哪一个预先设定的bbox形状与groundtruth bbox的形状最匹配
target_id[b, ind_y, ind_x, best_match, :] = l[
0] #### 将best_match的bbox的类别设置为该groudtruth bbox的类别
target_score[
b, ind_y, ind_x,
best_match, :] = 1.0 #将best_match的bbox的score赋为1，其他bbox的score都为零
tw = np.log(gw / anchors[best_match, 0]) #【【？？？？？？？？？？？？？？？？】】
th = np.log(gh / anchors[best_match, 1])
target_box[b, ind_y, ind_x, best_match, :] = mx.nd.array(
[tx, ty, tw, th]) #tx, ty, tw, th 即网络输出的四个坐标讯息
sample_weight[b, ind_y, ind_x, best_match, :] = 1.0
# print('ind_y', ind_y, 'ind_x', ind_x, 'best_match', best_match, 't', tx, ty, tw, th, 'ovp', ovps[best_match], 'gt', gx, gy, gw/w, gh/h, 'anchor', anchors[best_match, 0], anchors[best_match, 1])
return target_id, target_score, target_box, sample_weight
def hybrid_forward(self, F, score_gt, kernel_gt, score_pred,
training_masks, *args, **kwargs):
# cal ohem mask
selected_masks = []
for i in range(score_gt.shape[0]):
# cal for text region
selected_mask = self._ohem_single(score_gt[i:i + 1],
score_pred[i:i + 1],
training_masks[i:i + 1])
selected_masks.append(selected_mask)
selected_masks = F.concat(*selected_masks, dim=0)
s1, s2, s3, s4, s5, s6 = F.split(kernel_gt,
num_outputs=6,
axis=3,
squeeze_axis=True)
s1_pred, s2_pred, s3_pred, s4_pred, s5_pred, s6_pred, C_pred = F.split(
score_pred, num_outputs=7, axis=1, squeeze_axis=True)
self.pixel_acc = batch_pix_accuracy(C_pred, score_gt)
# for text map
eps = 1e-5
intersection = F.sum(score_gt * C_pred * selected_masks, axis=1)
union = F.sum(score_gt * selected_masks, axis=1) + F.sum(
C_pred * selected_mask, axis=1) + eps
C_dice_loss = 1. - F.mean((2 * intersection / union))
# loss for kernel
kernel_dices = []
for s, s_pred in zip(
[s1, s2, s3, s4, s5, s6],
[s1_pred, s2_pred, s3_pred, s4_pred, s5_pred, s6_pred]):
kernel_mask = F.where(C_pred > 0.5, F.ones_like(s_pred),
F.zeros_like(s_pred))
kernel_mask = F.cast(kernel_mask, dtype='float32')
kernel_mask = F.cast(F.logical_or(kernel_mask, score_gt),
dtype='float32')
s = F.cast(s, dtype='float32')
kernel_intersection = F.sum(s * s_pred * training_masks *
kernel_mask,
axis=1)
kernel_union = F.sum(
training_masks * s * kernel_mask, axis=1) + F.sum(
training_masks * s_pred * kernel_mask, axis=1) + eps
kernel_dice = 2. * kernel_intersection / kernel_union
kernel_dice = 1. - F.mean(
(2. * kernel_intersection / kernel_union))
kernel_dices.append(kernel_dice)
kernel_dice_loss = F.mean(F.array(kernel_dices))
self.kernel_loss = kernel_dice_loss
self.C_loss = C_dice_loss
loss = self.lam * C_dice_loss + (1. - self.lam) * kernel_dice_loss
return loss
def get_rmse(class_pre_l, class_true_l, con_pre_l, con_true_l, data_utils):
# find right predictions
one_zero_pre = nd.where(class_pre_l > 0.5, nd.ones_like(class_pre_l),
nd.zeros_like(class_pre_l))
compare = nd.equal(one_zero_pre, class_true_l).sum(axis=1)
weight_right = nd.repeat(nd.expand_dims(nd.where(compare == 3, nd.ones_like(compare), nd.zeros_like(compare)),\
axis=0),repeats=2,axis=0).transpose()
# calculate rmse based on right prediction
eth_co_me_limit = nd.array([[
data_utils.scale_CO[1], data_utils.scale_CO[0], data_utils.scale_Me[0]
]])
concentration_mat = nd.where(class_pre_l > 0.5, nd.repeat(eth_co_me_limit,repeats=class_pre_l.shape[0],axis=0), \
nd.zeros_like(class_pre_l))
eth_pre_con, eth_pre_con_true = concentration_mat[:,
0] * con_pre_l[:,
1], concentration_mat[:,
0] * con_true_l[:,
1]
co_pre_con, co_pre_con_true = concentration_mat[:,
1] * con_pre_l[:,
0], concentration_mat[:,
1] * con_true_l[:,
0]
me_pre_con, me_pre_con_true = concentration_mat[:,
2] * con_pre_l[:,
0], concentration_mat[:,
2] * con_true_l[:,
0]
co_or_me_con, co_or_me_con_true = co_pre_con + me_pre_con, co_pre_con_true + me_pre_con_true
co_or_me_eth_con = nd.concat(nd.expand_dims(co_or_me_con, axis=0),
nd.expand_dims(eth_pre_con, axis=0),
dim=0).transpose()
co_or_me_eth_con_true = nd.concat(nd.expand_dims(co_or_me_con_true,
axis=0),
nd.expand_dims(eth_pre_con_true, axis=0),
dim=0).transpose()
# rmse = (((co_or_me_eth_con-co_or_me_eth_con_true)**2*weight_right).sum()/(weight_right[:,0].sum()))**(0.5)
rmse = (((co_or_me_eth_con - co_or_me_eth_con_true)**2).mean(axis=0))
return rmse
def yolo2_target(scores, boxes, labels, anchors, ignore_label=-1, thresh=0.5):
b, h, w, n, _ = scores.shape # n: the number of scales of anchors
anchors = np.reshape(
np.array(anchors),
(-1, 2)) # numpy doesn't support autograde, anchors = (still 2 * 2)?
# define ground truth, labels = (score, top, left, right, bottom...)
#gt_boxes = nd.slice_axis(labels, begin=1, end=5, axis=-1)
target_score = nd.zeros(
(b, h, w, n, 1),
ctx=scores.context) # here, n = 2, the number of scales
target_id = nd.ones_like(target_score, ctx=scores.context) * ignore_label
target_box = nd.zeros((b, h, w, n, 4), ctx=scores.context)
sample_weight = nd.zeros((b, h, w, n, 1), ctx=scores.context)
#for i in range(output.shape[0]):
for i in range(b):
# find the bestmatch for each gt
label = labels[i].asnumpy() # labels -> (b, 1, 5), b: batch size
valid_label = label[np.where(label[:, 0] > -0.5)[0], :]
# shuffle because multi gt could possibily match to one anchor, we keep the last match randomly
np.random.shuffle(valid_label)
for l in valid_label:
gx, gy, gw, gh = (l[1] + l[3]) / 2, (
l[4] + l[2]) / 2, l[3] - l[1], l[4] - l[2]
ind_x = int(gx * w)
ind_y = int(gy * h)
tx = gx * w - ind_x
ty = gy * h - ind_y
gw = gw * w
gh = gh * h
# find the best match using width and height only, assuming centers are identical
# *: element-wise multiplication
#print('anchors = ', anchors) # 2 * 2
#print('anchors[:, 0] = ', anchors[:, 0]) # 2 * 1
# here, assuming centers are identical, the anchors only represent the width and height of two scales, so its shape is 2 * 2
intersect = np.minimum(anchors[:, 0], gw) * np.minimum(
anchors[:, 1], gh)
#print('intersec = ', intersect) # shape = 2 * 2
ovps = intersect / (gw * gh + anchors[:, 0] * anchors[:, 1] -
intersect)
#print('ovps = ', ovps)
best_match = int(np.argmax(ovps)) # the best match of scale
target_id[i, ind_y, ind_x, best_match, :] = l[0]
target_score[i, ind_y, ind_x, best_match, :] = 1.0
tw = np.log(
gw / anchors[best_match, 0]
) # return the relative width and height to the anchor, then compute the log
th = np.log(gh / anchors[best_match, 1])
#print('best match = ', best_match)
target_box[i, ind_y, ind_x,
best_match, :] = mx.nd.array([tx, ty, tw, th])
sample_weight[i, ind_y, ind_x, best_match, :] = 1.0
return target_id, target_score, target_box, sample_weight
def forward(self, x):
root = next(iter(self._structure.items()))[0]
if (len(self._routerlayer) > 0):
router_d, router_mat_d, weight_d, embedd_d = self._contextify(x)(
root)
# router = nd.stack(*[router_d[key] for key in sorted(router_d)], axis = -1)
# weight = nd.stack(*[weight_d[key] for key in sorted(weight_d)], axis = -1)
#
# embedd = nd.stack(*[embedd_d[key] for key in sorted(embedd_d)], axis = 0)
# router_mat = nd.stack(
# *[router_mat_d[key] for key in sorted(router_mat_d)], axis = 1)
#
# presence = nd.sum(router_mat, axis = 2)
# weight_adj = presence * weight
# depth = len(self._weightlayer) - nd.topk(nd.reverse(presence, axis = 1))
# depth = depth - 1
# depth = depth[:, 0]
# remainder = 1 - nd.sum(weight_adj, axis = 1)
#
# if (mx.autograd.is_training()):
# # remainder = remainder + nd.choose_element_0index(weight_adj, depth)
# remainder = remainder + nd.concat(
# *[x[d] for d, x in zip(depth, weight_adj)], dim = 0)
# # weight_adj = nd.fill_element_0index(weight_adj, remainder, depth)
# weight_adj = nd.stack(
# *[nd.concat(*[y if i != d else r for i, y in enumerate(x)], dim = 0)
# for d, r, x in zip(depth, remainder, weight_adj)
# ], axis = 0)
# else:
# remainder = remainder + nd.choose_element_0index(weight_adj, depth)
# weight_adj = nd.fill_element_0index(weight_adj, remainder, depth)
#
# head = nd.sum(nd.expand_dims(weight_adj, axis = 2) * router_mat, axis = 1)
#
# return nd.dot(head, embedd)
embedd = nd.stack(*[embedd_d[key] for key in sorted(embedd_d)],
axis=0)
router = nd.stack(*[router_d[key] for key in sorted(router_d)],
axis=-1)
router_mat = nd.stack(
*[router_mat_d[key] for key in sorted(router_mat_d)], axis=1)
where = nd.argmax(nd.maximum(0, 1 / (router + 0.5)), axis=1)
head = nd.concat(*[router_mat[i][k] for i, k in enumerate(where)],
dim=0)
return nd.dot(head, embedd)
else:
head = nd.ones_like(nd.slice_axis(x, axis=1, begin=0, end=None))
return self._contextify(x)(root) * head
def hybrid_forward(self, F, samples, matches, refs):
"""HybridBlock, handle multi batch correctly
Parameters
----------
samples: (B, N), value +1 (positive), -1 (negative), 0 (ignore)
matches: (B, N), value range [0, M)
refs: (B, M), value range [0, num_fg_class), excluding background
Returns
-------
targets: (B, N), value range [0, num_fg_class + 1), including background
"""
# samples (B, N) (+1, -1, 0: ignore), matches (B, N) [0, M), refs (B, M)
# reshape refs (B, M) -> (B, 1, M) -> (B, N, M)
refs = F.repeat(refs.reshape((0, 1, -1)), axis=1, repeats=matches.shape[1])
# ids (B, N, M) -> (B, N), value [0, M + 1), 0 reserved for background class
target_ids = F.pick(refs, matches, axis=2) + 1
# samples 0: set ignore samples to ignore_label
targets = F.where(samples > 0.5, target_ids, nd.ones_like(target_ids) * self._ignore_label)
# samples -1: set negative samples to 0
targets = F.where(samples < -0.5, nd.zeros_like(targets), targets)
return targets
def forward(self, pred, label, valid_length):
weights = nd.ones_like(label).expand_dims(axis=-1)
weights = nd.SequenceMask(weights, valid_length, True, axis=1)
return super(MaskedSoftmaxCELoss, self).forward(pred, label, weights)
def condition(y):
return nd.ones_like(y, ctx=y.context)