def forward(self, is_train, req, in_data, out_data, aux):
rois = in_data[-1].asnumpy()
w = rois[:, 3] - rois[:, 1] + 1
h = rois[:, 4] - rois[:, 2] + 1
#Leonid fix, just in case w / h will misbehave
w = np.maximum(w, 1e-7)
h = np.maximum(h, 1e-7)
feat_id = np.clip(np.floor(2 + np.log2(np.sqrt(w * h) / 224)), 0,
len(self.feat_strides) - 1)
pyramid_idx = []
rois_p = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
self.feat_idx[i] = np.where(feat_id == i)[0]
if len(self.feat_idx[i]) == 0:
# padding dummy roi
rois_p[i] = np.zeros((1, 5))
pyramid_idx.append(-1)
else:
rois_p[i] = rois[self.feat_idx[i]]
pyramid_idx.append(self.feat_idx[i])
rois_idx = np.argsort(np.hstack(pyramid_idx))[-rois.shape[0]:]
if is_train:
for i in range(self.num_strides):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
autograd.mark_variables(
[in_data[i] for i in range(self.num_strides)],
self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
self.roi_pool[i] = mx.nd.ROIPooling(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i])
rpn_conv = mx.sym.Convolution(
data=in_data[i],
kernel=(3, 3),
pad=(1, 1),
num_filter=512,
name='rpn_conv_' + suffix,
weight=self.shared_param_dict['rpn_conv_weight'],
bias=self.shared_param_dict['rpn_conv_bias'])
else:
roi_pool = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
roi_pool[i] = mx.nd.ROIPooling(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.take(roi_pool, mx.nd.array(rois_idx,
roi_pool.context))
self.assign(out_data[0], req[0], roi_pool)
def forward(self, is_train, req, in_data, out_data, aux):
rois = in_data[-1].asnumpy()
# w = rois[:, 3] - rois[:, 1] + 1
# h = rois[:, 4] - rois[:, 2] + 1
w = np.maximum(rois[:, 3], 1)
h = np.maximum(rois[:, 4], 1)
# TODO: carefully scale the w, h
feat_id = np.clip(np.floor(2 + np.log2(np.sqrt(w * h) / 224)), 0,
len(self.feat_strides) - 1)
pyramid_idx = []
rois_p = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
self.feat_idx[i] = np.where(feat_id == i)[0]
if len(self.feat_idx[i]) == 0:
# padding dummy roi
rois_p[i] = np.zeros((1, 6))
pyramid_idx.append(-1)
else:
rois_p[i] = rois[self.feat_idx[i]]
pyramid_idx.append(self.feat_idx[i])
rois_idx = np.argsort(np.hstack(pyramid_idx))[-rois.shape[0]:]
if is_train:
for i in range(self.num_strides):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
autograd.mark_variables(
[in_data[i] for i in range(self.num_strides)],
self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
self.roi_pool[i] = mx.nd.contrib.ROIAlignRotated(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i],
sample_ratio=4)
# self.roi_pool[i] = mx.nd.ROIPooling(in_data[i], mx.nd.array(rois_p[i], in_data[i].context), (7, 7), spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.concatenate(self.roi_pool, axis=0)
else:
# during testing, there is no need to record variable, thus saving memory
# pdb.set_trace()
roi_pool = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
# roi_pool[i] = mx.nd.ROIPooling(in_data[i], mx.nd.array(rois_p[i], in_data[i].context), (7, 7), spatial_scale=1.0 / self.feat_strides[i])
roi_pool[i] = mx.nd.contrib.ROIAlignRotated(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i],
sample_ratio=4)
roi_pool = mx.nd.concatenate(roi_pool, axis=0)
# pdb.set_trace()
roi_pool = mx.nd.take(roi_pool, mx.nd.array(rois_idx,
roi_pool.context))
self.assign(out_data[0], req[0], roi_pool)
def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
for i in range(len(in_grad)):
self.assign(in_grad[i], req[i], 0)
with autograd.train_section():
for i in range(self.num_strides):
if len(self.feat_idx[i] > 0):
autograd.compute_gradient([mx.nd.take(out_grad[0], mx.nd.array(self.feat_idx[i], out_grad[0].context)) * self.roi_pool[i]])
if self.with_deformable:
for i in range(0, self.num_strides * 3):
self.assign(in_grad[i], req[i], self.in_grad_hist_list[i])
else:
for i in range(0, self.num_strides):
self.assign(in_grad[i], req[i], self.in_grad_hist_list[i])
gc.collect()
def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
for i in range(len(in_grad)):
self.assign(in_grad[i], req[i], 0)
with autograd.train_section():
for i in range(self.num_strides):
if len(self.feat_idx[i] > 0):
autograd.compute_gradient([mx.nd.take(out_grad[0], mx.nd.array(self.feat_idx[i], out_grad[0].context)) * self.roi_pool[i]])
if self.with_deformable:
for i in range(0, self.num_strides * 3):
self.assign(in_grad[i], req[i], self.in_grad_hist_list[i])
else:
for i in range(0, self.num_strides):
self.assign(in_grad[i], req[i], self.in_grad_hist_list[i])
gc.collect()
def forward(self, is_train, req, in_data, out_data, aux):
rois = in_data[-1].asnumpy()
w = rois[:, 3] - rois[:, 1] + 1
h = rois[:, 4] - rois[:, 2] + 1
#Leonid fix, just in case w / h will misbehave
w = np.maximum(w, 1e-7)
h = np.maximum(h, 1e-7)
feat_id = np.clip(np.floor(2 + np.log2(np.sqrt(w * h) / 224)), 0,
len(self.feat_strides) - 1)
pyramid_idx = []
rois_p = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
self.feat_idx[i] = np.where(feat_id == i)[0]
if len(self.feat_idx[i]) == 0:
# padding dummy roi
rois_p[i] = np.zeros((1, 5))
pyramid_idx.append(-1)
else:
rois_p[i] = rois[self.feat_idx[i]]
pyramid_idx.append(self.feat_idx[i])
rois_idx = np.argsort(np.hstack(pyramid_idx))[-rois.shape[0]:]
if is_train:
for i in range(self.num_strides):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
if self.with_deformable:
for i in range(self.num_strides, self.num_strides * 3):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
autograd.mark_variables(
[in_data[i] for i in range(self.num_strides * 3)],
self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(
data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=1,
pooled_size=7,
sample_per_part=4,
no_trans=True,
part_size=7,
output_dim=256,
spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(
data=roi_offset_t,
num_hidden=7 * 7 * 2,
weight=in_data[i * 2 + self.num_strides],
bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset,
shape=(-1, 2, 7, 7))
self.roi_pool[
i] = mx.contrib.nd.DeformablePSROIPooling(
data=in_data[i],
rois=mx.nd.array(rois_p[i],
in_data[i].context),
trans=roi_offset_reshape,
group_size=1,
pooled_size=7,
sample_per_part=4,
no_trans=False,
part_size=7,
output_dim=self.output_dim,
spatial_scale=1.0 / self.feat_strides[i],
trans_std=0.1)
else:
autograd.mark_variables(
[in_data[i] for i in range(self.num_strides)],
self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
self.roi_pool[i] = mx.nd.ROIPooling(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.concatenate(self.roi_pool, axis=0)
else:
# during testing, there is no need to record variable, thus saving memory
roi_pool = [None for _ in range(self.num_strides)]
if self.with_deformable:
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(
data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=1,
pooled_size=7,
sample_per_part=4,
no_trans=True,
part_size=7,
output_dim=256,
spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(
data=roi_offset_t,
num_hidden=7 * 7 * 2,
weight=in_data[i * 2 + self.num_strides],
bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset,
shape=(-1, 2, 7, 7))
roi_pool[i] = mx.contrib.nd.DeformablePSROIPooling(
data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
trans=roi_offset_reshape,
group_size=1,
pooled_size=7,
sample_per_part=4,
no_trans=False,
part_size=7,
output_dim=self.output_dim,
spatial_scale=1.0 / self.feat_strides[i],
trans_std=0.1)
else:
for i in range(self.num_strides):
roi_pool[i] = mx.nd.ROIPooling(
in_data[i],
mx.nd.array(rois_p[i], in_data[i].context), (7, 7),
spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.concatenate(roi_pool, axis=0)
roi_pool = mx.nd.take(roi_pool, mx.nd.array(rois_idx,
roi_pool.context))
self.assign(out_data[0], req[0], roi_pool)
def forward(self, is_train, req, in_data, out_data, aux):
rois = in_data[-1].asnumpy()
w = rois[:, 3] - rois[:, 1] + 1
h = rois[:, 4] - rois[:, 2] + 1
feat_id = np.clip(np.floor(2 + np.log2(np.sqrt(w * h) / 224)), 0, len(self.feat_strides) - 1)
pyramid_idx = []
rois_p = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
self.feat_idx[i] = np.where(feat_id == i)[0]
if len(self.feat_idx[i]) == 0:
# padding dummy roi
rois_p[i] = np.zeros((1, 5))
pyramid_idx.append(-1)
else:
rois_p[i] = rois[self.feat_idx[i]]
pyramid_idx.append(self.feat_idx[i])
rois_idx = np.argsort(np.hstack(pyramid_idx))[-rois.shape[0]:]
if is_train:
for i in range(self.num_strides):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
if self.with_deformable:
for i in range(self.num_strides, self.num_strides * 3):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
autograd.mark_variables([in_data[i] for i in range(self.num_strides * 3)], self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), group_size=1, pooled_size=7,
sample_per_part=4, no_trans=True, part_size=7, output_dim=256, spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(data=roi_offset_t, num_hidden=7 * 7 * 2, weight=in_data[i * 2 + self.num_strides], bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset, shape=(-1, 2, 7, 7))
self.roi_pool[i] = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), trans=roi_offset_reshape,
group_size=1, pooled_size=7, sample_per_part=4, no_trans=False, part_size=7,
output_dim=self.output_dim, spatial_scale=1.0 / self.feat_strides[i], trans_std=0.1)
else:
autograd.mark_variables([in_data[i] for i in range(self.num_strides)], self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
self.roi_pool[i] = mx.nd.ROIPooling(in_data[i], mx.nd.array(rois_p[i], in_data[i].context), (7, 7), spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.concatenate(self.roi_pool, axis=0)
else:
# during testing, there is no need to record variable, thus saving memory
roi_pool = [None for _ in range(self.num_strides)]
if self.with_deformable:
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), group_size=1, pooled_size=7,
sample_per_part=4, no_trans=True, part_size=7, output_dim=256, spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(data=roi_offset_t, num_hidden=7 * 7 * 2, weight=in_data[i * 2 + self.num_strides], bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset, shape=(-1, 2, 7, 7))
roi_pool[i] = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), trans=roi_offset_reshape,
group_size=1, pooled_size=7, sample_per_part=4, no_trans=False, part_size=7,
output_dim=self.output_dim, spatial_scale=1.0 / self.feat_strides[i], trans_std=0.1)
else:
for i in range(self.num_strides):
roi_pool[i] = mx.nd.ROIPooling(in_data[i], mx.nd.array(rois_p[i], in_data[i].context), (7, 7), spatial_scale=1.0 / self.feat_strides[i])
roi_pool = mx.nd.concatenate(roi_pool, axis=0)
roi_pool = mx.nd.take(roi_pool, mx.nd.array(rois_idx, roi_pool.context))
self.assign(out_data[0], req[0], roi_pool)
def forward(self, is_train, req, in_data, out_data, aux):
rois = in_data[-1].asnumpy()
w = rois[:, 3] - rois[:, 1] + 1
h = rois[:, 4] - rois[:, 2] + 1
feat_id = np.clip(np.floor(2 + np.log2(np.sqrt(w * h) / 224)), 0, len(self.feat_strides) - 1)
pyramid_idx = []
rois_p = [None for _ in range(self.num_strides)]
for i in range(self.num_strides):
self.feat_idx[i] = np.where(feat_id == i)[0]
if len(self.feat_idx[i]) == 0:
# padding dummy roi
rois_p[i] = np.zeros((1, 5))
pyramid_idx.append(-1)
else:
rois_p[i] = rois[self.feat_idx[i]]
pyramid_idx.append(self.feat_idx[i])
rois_idx = np.argsort(np.hstack(pyramid_idx))[-rois.shape[0]:]
# pdb.set_trace()
if is_train:
for i in range(self.num_strides):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
if self.pooling_mode == 'deform':
for i in range(self.num_strides, self.num_strides * 3):
self.in_grad_hist_list.append(mx.nd.zeros_like(in_data[i]))
autograd.mark_variables([in_data[i] for i in range(self.num_strides * 3)], self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), group_size=7, pooled_size=7,
sample_per_part=4, no_trans=True, part_size=7, output_dim=10, spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(data=roi_offset_t, num_hidden=7 * 7 * 2, weight=in_data[i * 2 + self.num_strides], bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset, shape=(-1, 2, 7, 7))
self.roi_pool[i] = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), trans=roi_offset_reshape,
group_size=7, pooled_size=7, sample_per_part=4, no_trans=False, part_size=7,
output_dim=self.output_dim, spatial_scale=1.0 / self.feat_strides[i], trans_std=0.1)
elif self.pooling_mode == 'alignave':
# pdb.set_trace()
autograd.mark_variables([in_data[i] for i in range(self.num_strides)], self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
self.roi_pool[i] = mx.contrib.nd.PSROIALIGNAVEPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=7, pooled_size=7, sampling_ratio=4,
output_dim=10,
spatial_scale=1.0 / self.feat_strides[i])
elif self.pooling_mode == 'pooling':
autograd.mark_variables([in_data[i] for i in range(self.num_strides)], self.in_grad_hist_list)
with autograd.train_section():
for i in range(self.num_strides):
# TODO: finish it, and fix the output_dim hard code here
# self.roi_pool[i] = mx.nd.contrib.PSROIPooling(data=in_data[i], rois=mx.nd.array(rois_p[i], in_data[i].context), group_size=7, pooled_size=7,
# output_dim=10, spatial_scale=1.0 / self.feat_strides[i])
self.roi_pool[i] = mx.contrib.nd.PSROIPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=7, pooled_size=7,
output_dim=10,
spatial_scale=1.0 / self.feat_strides[i])
else:
print 'no such pooling mode'
pdb.set_trace()
roi_pool = mx.nd.concatenate(self.roi_pool, axis=0)
else:
# during testing, there is no need to record variable, thus saving memory
# pdb.set_trace()
roi_pool = [None for _ in range(self.num_strides)]
if self.pooling_mode == 'deform':
for i in range(self.num_strides):
roi_offset_t = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=7, pooled_size=7,
sample_per_part=4, no_trans=True, part_size=7,
output_dim=10,
spatial_scale=1.0 / self.feat_strides[i])
roi_offset = mx.nd.FullyConnected(data=roi_offset_t, num_hidden=7 * 7 * 2,
weight=in_data[i * 2 + self.num_strides],
bias=in_data[i * 2 + 1 + self.num_strides])
roi_offset_reshape = mx.nd.reshape(data=roi_offset, shape=(-1, 2, 7, 7))
roi_pool[i] = mx.contrib.nd.DeformablePSROIPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
trans=roi_offset_reshape,
group_size=7, pooled_size=7, sample_per_part=4,
no_trans=False, part_size=7,
output_dim=self.output_dim,
spatial_scale=1.0 / self.feat_strides[i],
trans_std=0.1)
elif self.pooling_mode == 'alignave':
for i in range(self.num_strides):
roi_pool[i] = mx.contrib.nd.PSROIALIGNAVEPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=7, pooled_size=7, sampling_ratio=4,
output_dim=10,
spatial_scale=1.0 / self.feat_strides[i])
elif self.pooling_mode == 'pooling':
for i in range(self.num_strides):
roi_pool[i] = mx.contrib.nd.PSROIPooling(data=in_data[i],
rois=mx.nd.array(rois_p[i], in_data[i].context),
group_size=7, pooled_size=7,
output_dim=10,
spatial_scale=1.0 / self.feat_strides[i])
else:
print 'no such pooling mode'
pdb.set_trace()
roi_pool = mx.nd.concatenate(roi_pool, axis=0)
roi_pool = mx.nd.take(roi_pool, mx.nd.array(rois_idx, roi_pool.context))
self.assign(out_data[0], req[0], roi_pool)
for epoch in range(opt.niter):
train_iter.reset()
for batch in train_iter:
train_G = sched.next()
############################
# (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
###########################
# train with real_t
data = batch.data[0].copyto(ctx) / 255. * 2. - 1.
noise = mx.nd.random_normal(0,
1,
shape=(opt.batchSize, nz, 1, 1),
ctx=ctx)
with autograd.train_section():
errD_real = netD(data)
#output = output.reshape((opt.batchSize, 2))
#errD_real = nn.loss.softmax_cross_entropy_loss(output, real_label)
fake = netG(noise)
errD_fake = netD(fake.detach())
#output = output.reshape((opt.batchSize, 2))
#errD_fake = nn.loss.softmax_cross_entropy_loss(output, fake_label)
errD = errD_real - errD_fake
errD.backward()
optimizerD.step(opt.batchSize)
for p in netD.params.values():
p.set_data(mx.nd.clip(p.data(ctx=ctx), -0.01, 0.01))
print('D: %.6f (%.6f, %.6f)' %
return train, val
train_iter, val_iter = cifar10_iterator(opt.batchSize, (3, 64, 64), 64)
for epoch in range(opt.niter):
train_iter.reset()
for batch in train_iter:
train_G = sched.next()
############################
# (1) Update D network: maximize log(D(x)) + log(1 - D(G(z)))
###########################
# train with real_t
data = batch.data[0].copyto(ctx) / 255. * 2. - 1.
noise = mx.nd.random_normal(0, 1, shape=(opt.batchSize, nz, 1, 1), ctx=ctx)
with autograd.train_section():
errD_real = netD(data)
#output = output.reshape((opt.batchSize, 2))
#errD_real = nn.loss.softmax_cross_entropy_loss(output, real_label)
fake = netG(noise)
errD_fake = netD(fake.detach())
#output = output.reshape((opt.batchSize, 2))
#errD_fake = nn.loss.softmax_cross_entropy_loss(output, fake_label)
errD = errD_real - errD_fake
errD.backward()
optimizerD.step(opt.batchSize)
for p in netD.params.values():
p.set_data(mx.nd.clip(p.data(ctx=ctx), -0.01, 0.01))
print('D: %.6f (%.6f, %.6f)' % (
