def show_img(img):
"""
Show an image or images of numpy or ndarray type
"""
if type(img) is np.ndarray:
cv2.imshow('a', img)
k = cv2.waitKey(0)
if k == ord('q'):
cv2.destroyAllWindows()
else:
if len(img.shape) == 3:
img = nd.swapaxes(img, 0, 1)
img = nd.swapaxes(img, 1, 2)
img = img.asnumpy()
cv2.imshow('a', img)
k = cv2.waitKey(0)
if k == ord('q'):
cv2.destroyAllWindows()
else:
for i in range(img.shape[0]):
cur_img = img[i]
cur_img = nd.swapaxes(cur_img, 0, 1)
cur_img = nd.swapaxes(cur_img, 1, 2)
cur_img = cur_img.asnumpy()
cv2.imshow('a', cur_img)
k = cv2.waitKey(0)
if k == ord('q'):
cv2.destroyAllWindows()
def next(self):
if self.curr_idx == len(self.idx):
raise StopIteration
if isinstance(self.sampling, list):
i, j = self.idx[self.curr_idx]
slice_end = min(self.nddata[i].shape[0], j + self.batch_size)
#slice_end = j + self.batch_size
data = self.nddata[i][j:slice_end]
label = self.ndlabel[i][j:slice_end]
names = self.names[i][j:slice_end]
lens = self.lengths[i][j:slice_end]
data = ndarray.swapaxes(data, 0, 1) # TBD
label = ndarray.swapaxes(label, 0, 1)
batch = UtteranceBatch([self.data_name], [data], [self.label_name],
[label],
bucket_key=self.sampling[i],
utt_names=names,
utt_lens=lens)
else:
assert False
self.curr_idx += 1
return batch
def _read_img(self, img_name):
img_np = cv2.imread(os.path.join(self.img_root, img_name))
img_np = self._resize_pad(img_np)
img_nd = nd.array(img_np, ctx=mx.cpu())
img_nd = nd.swapaxes(img_nd, 1, 2)
img_nd = nd.swapaxes(img_nd, 0, 1)
trans = transforms.Normalize(0.13, 0.31)
img_nd = trans(img_nd)
return img_nd
def __getitem__(self, idx):
img_np = cv2.imread(os.path.join(self.img_root, self.img_list[idx]))
img_nd = nd.array(img_np)
img_nd = nd.swapaxes(img_nd, 1, 2)
img_nd = nd.swapaxes(img_nd, 0, 1)
temp = self.img_list[idx].split('.')[0]
class_id = int(temp.split('/')[0])
img_id = int(temp.split('/')[1])
return img_nd, class_id, img_id
def subtract_imagenet_mean_preprocess_batch(batch):
"""Subtract ImageNet mean pixel-wise from a BGR image."""
batch = F.swapaxes(batch,0, 1)
(r, g, b) = F.split(batch, num_outputs=3, axis=0)
r = r - 123.680
g = g - 116.779
b = b - 103.939
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch,0, 1)
return batch
def subtract_imagenet_mean_preprocess_batch(batch):
"""Subtract ImageNet mean pixel-wise from a BGR image."""
batch = F.swapaxes(batch, 0, 1)
(r, g, b) = F.split(batch, num_outputs=3, axis=0)
r = r - 123.680
g = g - 116.779
b = b - 103.939
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch, 0, 1)
return batch
def __getitem__(self, idx):
img_np = cv2.imread(os.path.join(self.img_root, self.img_list[idx]))
img_np = resize_pad(img_np, self.img_size)
img_nd = nd.array(img_np)
img_nd = nd.swapaxes(img_nd, 1, 2)
img_nd = nd.swapaxes(img_nd, 0, 1)
temp = self.img_list[idx]
class_id = int(self.img_map[temp])
img_id = idx
return img_nd, class_id
def add_imagenet_mean_batch(batch):
batch = F.swapaxes(batch, 0, 1)
(b, g, r) = F.split(batch, num_outputs=3, axis=0)
r = r + 123.680
g = g + 116.779
b = b + 103.939
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch, 0, 1)
"""
batch = denormalizer(batch)
"""
return batch
def add_imagenet_mean_batch(batch):
batch = F.swapaxes(batch,0, 1)
(b, g, r) = F.split(batch, num_outputs=3, axis=0)
r = r + 123.680
g = g + 116.779
b = b + 103.939
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch,0, 1)
"""
batch = denormalizer(batch)
"""
return batch
def vclip_2_ndarray(in_file, label, out_file):
""" Convert a video file to mxnet.ndarray array
"""
vframe_list = []
cap = cv2.VideoCapture(in_file)
while(cap.isOpened()):
ret, vframe = cap.read()
if not ret:
break
vframe = cv2.cvtColor(vframe, cv2.COLOR_BGR2RGB)
vframe = cv2.resize(vframe, (171, 128))
vframe = nd.array(vframe)
vframe,_ = mx.img.center_crop(vframe, (112, 112))
vframe = nd.swapaxes(vframe, 0, 2)
vframe = nd.expand_dims(vframe, axis=1)
vframe_list.append(vframe)
if len(vframe_list) is not 0:
v_arr = nd.concat(*vframe_list, dim=1)
label_arr = nd.empty((1,1))
label_arr[0] = label
data = {"data": v_arr.astype(np.uint8), "label": label_arr}
# nd.save(out_file, data)
else:
raise ValueError("File %s Open Failed", in_file)
def Mask(inputs, seq_len=None, mode='mul'):
if seq_len is None:
return inputs
if mode == 'mul':
value = 0
else:
value = -1e12
inputs = nd.swapaxes(inputs, 0, 1)
inputs_masked = nd.SequenceMask(inputs,
sequence_length=seq_len,
use_sequence_length=True,
value=value)
inputs_masked = nd.swapaxes(inputs_masked, 0, 1)
return inputs_masked
def forward(self, x, qst):
with x.context:
self.coord_tensor = F.zeros((x.shape[0], 25, 2))
# prepare coord tensor
def cvt_coord(i):
return [(i / 5 - 2) / 2., (i % 5 - 2) / 2.]
for i in range(25):
self.coord_tensor[:, i, :] = F.array(cvt_coord(i))
#input size = (64 * 3 * 75 * 75)
x = self.conv(x) ## x = (64 * 24 * 5 * 5)
##g part
mb = x.shape[0]
n_channels = x.shape[1]
d = x.shape[2]
x_flat = x.reshape(shape=(mb, n_channels, d * d))
x_flat = F.swapaxes(x_flat, 1, 2) ## (64 * 25 * 24)
##add coordinates
x_flat = F.concat(x_flat, self.coord_tensor, dim=2)
##add question
qst = qst.expand_dims(1)
qst = F.repeat(qst, repeats=25, axis=1)
qst = qst.expand_dims(2)
# cast all pairs against each other
x_i = x_flat.expand_dims(1)
x_i = F.repeat(x_i, repeats=25, axis=1)
x_j = x_flat.expand_dims(2)
x_j = F.concat(x_j, qst, dim=3)
x_j = F.repeat(x_j, repeats=25, axis=2)
#concatenate all
x_full = F.concat(x_i, x_j, dim=3)
#reshape and apply dnn network
x_ = x_full.reshape((-1, 63))
x_ = self.g_fc1(x_)
x_ = self.g_fc2(x_)
x_ = self.g_fc3(x_)
x_ = self.g_fc4(x_)
x_g = x_.reshape((mb, -1, 256))
x_g = x_g.sum(1)
##### f part #######
x_f = self.f_fc1(x_g)
return self.fcout(x_f)
def var(array,W=_W,B=None,square=0,sqrt=0,V=False,order='NCHW',sizz=0):
arrs=array.shape
ashp=W.shape
xi=(-2,-1)
x2=(-2,-1,-3)
sb=(ashp[1],1,1)
WV=ashp[-2:]
print(sb)
mnc=mnd.tile(mnd.reshape(mnd.array([WV[0]*WV[1]]), shape=(1,1,1)),ashp[1])
print(mnc)
if V:
print(W.eval())
print(arrs,ashp)
mul=(mnd.broadcast_mul(array,W))
if V:
print('Wsamp',W[-1,-1])
print('array*w',mul[0,-1])
size=mnd.sum(W,axis=xi,keepdims=True)#shape=(outputs, channel)
if V:
print("sizesamp",size.shape,size)
if B is None:
B=mnd.zeros(W.shape[0:2],dtype=np.float32)#channel
B=mnd.reshape(B,(*B.shape,*[1 for _ in range(len(ashp)-len(B.shape))]))
if sizz==1:
mean=mnd.sum(mul,axis=xi,keepdims=True)/size
else:
mean=mnd.sum(mul,axis=xi,keepdims=True)/mnc
if V:
print("meansamp",mean[0,-1])
if square:
i=mnd.square(mnd.broadcast_add(mnd.broadcast_minus(mul,mean),B))
else:
i=mnd.broadcast_add(mnd.broadcast_minus(mul,mean),B)
di=i/size
if V==2:
print("i",i,"i")
print("di",di,"di")
if V:
print('isamp',i.shape,i[-1,-1,])
out=mnd.sum(mnd.broadcast_add(i,B),axis=x2)
#out=np.rollaxis(np.sum(i+B,axis=x2),-1,1)
#print(out.shape)
if sqrt:
out=mnd.sqrt(out)
out=mnd.swapaxes(out, 3, 1)
#print(out.shape,(arrs[0],ashp[0],arrs[1],arrs[2]))
assert out.shape==(arrs[0],ashp[0],arrs[1],arrs[2])
return(out)
def hybrid_forward(self, F, x, *args, **kwargs):
# batch_size, channels, height, width = x.shape
# assert channels % 2 == 0
# mid_channels = channels // 2
data = F.reshape(x, shape=(0, -4, self.groups, -1, -2))
data = F.swapaxes(data, 1, 2)
data = F.reshape(data, shape=(0, -3, -2))
data_project = F.slice(data,
begin=(None, None, None, None),
end=(None, self.mid_channel, None, None))
data_x = F.slice(data,
begin=(None, self.mid_channel, None, None),
end=(None, None, None, None))
return data_project, data_x
def hybrid_forward(self, F, data):
# data.shape is a triple of (16, 1, 64). Need to eliminate that redundant second dimension and transpose it
# before attaching the embeddings
data = squeeze(data)
data = data.T
embedded = self.embedding(data)
x = self.rnn(embedded)
x - self.activation1(x)
# Swap the first and second axes to bring it from (length, batch size, width) to (batch size, length, width),
# before passing it to the outer layer (only recurrent layers use the first ordering).
x = swapaxes(x, 0, 1)
x = self.out(x)
return x
def swapaxes(input, axis1, axis2):
return nd.swapaxes(input, axis1, axis2)
def preprocess_batch(batch):
batch = F.swapaxes(batch, 0, 1)
(r, g, b) = F.split(batch, num_outputs=3, axis=0)
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch, 0, 1)
return batch
def np2nd(img_np, ctx=get_ctx()):
img_nd = nd.array(img_np, ctx=ctx)
img_nd = nd.swapaxes(img_nd, 1, 2)
img_nd = nd.swapaxes(img_nd, 0, 1)
img_nd = nd.expand_dims(img_nd, 0)
return img_nd
def preprocess_batch(batch):
batch = F.swapaxes(batch, 0, 1)
(r, g, b) = F.split(batch, num_outputs=3, axis=0)
batch = F.concat(b, g, r, dim=0)
batch = F.swapaxes(batch, 0, 1)
return batch