def __init__(self, vsize, ctx='cpu'):
self.vsize = vsize
self.ctx = mx.gpu() if ctx == 'gpu' else mx.cpu()
self.W1 = nd.random_uniform(-.01,
.01,
shape=(vsize + 1024, 1024),
ctx=self.ctx)
self.b1 = nd.zeros(1024, ctx=self.ctx)
self.W2 = nd.random_uniform(-.01,
.01,
shape=(2048, 1024),
ctx=self.ctx)
self.b2 = nd.zeros(1024, ctx=self.ctx)
self.Wy = nd.random_uniform(-.01,
.01,
shape=(1024, vsize),
ctx=self.ctx)
self.by = nd.zeros(vsize, ctx=self.ctx)
self.params = [self.W1, self.b1, self.W2, self.b2, self.Wy, self.by]
for p in self.params:
p.attach_grad()
def lkm_test():
net = LKM(pretrained=True)
net.collect_params().initialize(init=mx.initializer.Xavier(magnitude=2.0),
ctx=cfg.ctx)
x = nd.random_uniform(shape=(2, 3, 512, 512))
y = net(x)
print(y)
def deeplab_test():
net = DeepLab(pretrained=False)
net.collect_params().initialize(init=mx.initializer.Xavier(magnitude=2.0),
ctx=cfg.ctx)
x = nd.random_uniform(shape=(2, 3, 448, 448))
y = net(x)
print(y)
def TimeseriesFromPSD_nd(param_noise):
"""
GPU only
"""
(asd_pos, asd_neg, low_f, high_f, high_f_, size, fs, fmin, fmax) = param_noise
(*D_, N) = size
D = reduce(lambda x, y: x * y, D_)
# Gauss noise and its one-sided PSD without window
gauss_noise = 1* nd.random_normal(loc=0,scale=64,shape=(D, N), ctx=ctx)
_, xf_noise, psd_gauss = oneSidedPeriodogram_nd(gauss_noise, fs=8192)
psd_gauss = nd.array(psd_gauss, ctx = ctx, dtype='float64')
psd_twosided = nd.concat( # low positive
nd.zeros((D, low_f), ctx = ctx, dtype='float64'),
# high positive
psd_gauss[:, low_f:high_f] * asd_pos,
nd.zeros((D, high_f_), ctx = ctx, dtype='float64'),
nd.zeros((D, high_f_), ctx = ctx, dtype='float64'),
# high negative
psd_gauss[:, low_f:high_f][::-1] * asd_neg,
# low negative
nd.zeros((D, low_f), ctx = ctx, dtype='float64'), dim=1)
amplitude = nd.sqrt(psd_twosided *2 *fs*N )
epsilon_imag = nd.random_uniform(low=0, high=1, shape=(D,N),ctx=ctx,dtype='float64')*2*np.pi
re = nd.cos(epsilon_imag)*amplitude
im = nd.sin(epsilon_imag)*amplitude
temp = nd.zeros((D, N*2) , ctx=ctx)
temp[:,::2] = re
temp[:,1::2] = im
timeseries = mx.contrib.ndarray.ifft(temp)/N
return timeseries.reshape(size), psd_twosided
def __init__(self, n_vis=2, n_hid=1, n_val=2, ctx=mx.cpu()):
self.n_vis = n_vis # num of units in visible (input) layer
self.n_hid = n_hid # num of units in hidden layer
self.n_val = n_val # num of values for each node
self.n_node = n_vis * n_val # each visnode has n_val nodes
self.ctx = ctx
a = sqrt(6. / (self.n_vis * n_val + n_hid))
self.W = nd.random_uniform(low=-a,
high=a,
shape=(self.n_node, n_hid),
ctx=self.ctx)
self.hb = nd.zeros([n_hid], ctx=self.ctx) # initialize h bias 0
self.vb = nd.zeros([self.n_node], ctx=self.ctx) # initialize v bias 0
self.dW = nd.zeros([self.n_node, n_hid], ctx=self.ctx)
self.dh = nd.zeros([n_hid], ctx=self.ctx)
self.dv = nd.zeros([self.n_node], ctx=self.ctx)
#for KL
self.enum_states = None
self.prob_states = None
self.prob_RGs = None
self.L_coeff = 0.0
self.M_coeff = 0.0
self.no_softmax = False
def shuffle_data_nd(data, peak_samppoint, peak_time, times):
shift_list = nd.random_uniform(0, peak_samppoint - round((peak_time-0.2)*data.shape[-1]), shape=(times), ctx=mx.cpu())
base = forward_moving_wave_nd(data, int(shift_list.asnumpy()[0]))
if times == 1:
return base, shift_list
else:
for shift_size in shift_list[1:]:
temp = forward_moving_wave_nd(data, int(shift_size.asnumpy()[0]))
base = nd.concatenate([base, temp] , axis = 0)
return base, shift_list
def query(self, images):
if self.pool_size == 0:
return images
ret_imgs = []
for i in range(images.shape[0]):
image = nd.expand_dims(images[i], axis=0)
if self.num_imgs < self.pool_size:
self.num_imgs = self.num_imgs + 1
self.images.append(image)
ret_imgs.append(image)
else:
p = nd.random_uniform(0, 1, shape=(1,)).asscalar()
if p > 0.5:
random_id = nd.random_uniform(0, self.pool_size - 1, shape=(1,)).astype(np.uint8).asscalar()
tmp = self.images[random_id].copy()
self.images[random_id] = image
ret_imgs.append(tmp)
else:
ret_imgs.append(image)
ret_imgs = nd.concat(*ret_imgs, dim=0)
return ret_imgs
def forward(self, is_train, req, in_data, out_data, aux):
x = in_data[0]
if is_train:
self._spatial_dropout_mask = F.broadcast_greater(
F.random_uniform(low=0, high=1, shape=(1, self._num_filters, 1, 1), ctx=self._ctx),
F.ones(shape=(1, self._num_filters, 1, 1), ctx=self._ctx) * self._p,
ctx=self._ctx
)
y = F.broadcast_mul(x, self._spatial_dropout_mask, ctx=self._ctx) / (1-self._p)
self.assign(out_data[0], req[0], y)
else:
self.assign(out_data[0], req[0], x)
def dropout(batch_X, drop_probability):
keep_probability = 1 - drop_probability
assert 0 <= keep_probability <= 1
if keep_probability == 0:
return batch_X.zeros_like()
# > 保存的概率才能够保留该样本该神经元的输出
mask = nd.random_uniform(
0, 1.0, batch_X.shape, ctx=batch_X.context) < keep_probability
# 保证 E[dropout(batch_X)] == batch_X
scale = 1 / keep_probability
return mask * batch_X * scale
def __init__(self, vsize, ctx='cpu'):
self.vsize = vsize
self.ctx = mx.gpu() if ctx == 'gpu' else mx.cpu()
self.a1 = init_gru_args(vsize, 1024, self.ctx)
self.Wy = nd.random_uniform(-.01, .01, shape=(1024, vsize), ctx=self.ctx)
self.by = nd.zeros(vsize, ctx=self.ctx)
self.params = [*self.a1, self.Wy, self.by]
for p in self.params:
p.attach_grad()
def init_lstm_args(xsize, hsize, ctx):
return (
nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx),
nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx),
nd.zeros(hsize, ctx=ctx),
nd.zeros(hsize, ctx=ctx),
nd.zeros(hsize, ctx=ctx),
nd.zeros(hsize, ctx=ctx)
)
def test_large_models():
ctx = default_context()
# Create model
net = gluon.nn.HybridSequential()
largest_num_features = 256
net.add(nn.Conv2D(largest_num_features, 3))
net.hybridize()
net.initialize(mx.init.Normal(sigma=0.01), ctx=ctx)
# Compute the height (=width) of the square tensor of the given size in bytes
def tensor_size(big_tensor_bytes):
bytes_per_float = 4
sz = int(
math.sqrt(big_tensor_bytes / largest_num_features /
bytes_per_float))
return (sz // 100) * 100
# The idea is to create models with large tensors of (say) 20% of the total memory.
# This in the past has given cudnnFind() trouble when it needed to allocate similar I/O's
# from the area carved out by the MXNET_GPU_MEM_POOL_RESERVE setting (by default 5%).
(free_mem_bytes,
total_mem_bytes) = mx.context.gpu_memory_info(ctx.device_id)
# This test needs to be 'qualified' for use with each new larger memory size
largest_supported_total_mem_GB = 32
if (total_mem_bytes > largest_supported_total_mem_GB * 1024 * 1024 * 1024):
sys.stderr.write(
' bypassing test due to too-large global memory of size {} ... '.
format(total_mem_bytes))
return
start_size = tensor_size(0.20 * total_mem_bytes)
num_trials = 10
sys.stderr.write(
' testing global memory of size {} ... '.format(total_mem_bytes))
sys.stderr.flush()
for i in range(num_trials):
sz = start_size - 10 * i
(height, width) = (sz, sz)
sys.stderr.write(" {}x{} ".format(height, width))
sys.stderr.flush()
data_in = nd.random_uniform(low=0,
high=255,
shape=(1, 3, height, width),
ctx=ctx,
dtype="float32")
# Evaluate model
net(data_in).asnumpy()
def daydream(self, n, k=1):
sys.stderr.write("Day dreaming ...\n")
h0 = nd.random_uniform(low=0,
high=1,
shape=[1, self.n_hid],
ctx=self.ctx)
for i in range(100):
v_prob, v = self.sample_v_given_h(h0)
h_prob, h0 = self.sample_h_given_v(v)
for i in range(n):
for cdk in range(k):
v_prob, v = self.sample_v_given_h(h0)
h_prob, h0 = self.sample_h_given_v(v)
ndx = nd.argmax(v.reshape([-1, self.n_val]), axis=1)
for n in ndx.asnumpy():
sys.stdout.write(str(int(n)))
sys.stdout.write("\n")
return
def sample_v_given_h_without_softmax(self, h0):
v1_prob = nd.sigmoid(self.propdown(h0))
v1 = v1_prob > nd.random_uniform(
low=0.0, high=1.0, shape=v1_prob.shape, ctx=self.ctx)
return [v1_prob, v1]
def sample_h_given_v(self, v0):
h1_prob = self.propup(v0)
h1 = h1_prob > nd.random_uniform(
low=0.0, high=1.0, shape=h1_prob.shape, ctx=self.ctx)
return [h1_prob, h1]
def __init__(self, **kwargs):
super(FancyBlock, self).__init__(**kwargs)
with self.name_scope():
self.dense = gluon.nn.Dense(256)
self.weight = nd.random_uniform(shape=(256, 20))
from mxnet import gluon
from mxnet import ndarray as nd
from mxnet import autograd
from mxnet.gluon import data as gdata
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
import mxnet
losser = gluon.loss.L2Loss()
X = nd.random_uniform(-100, 100, shape=(1000))
Y = nd.random_uniform(-100, 100, shape=(1000))
XY = nd.concat(X.reshape((-1, 1)), Y.reshape((-1, 1)), dim=1)
Z = nd.exp(-nd.abs(X - Y))
figure = plt.figure()
axes = plt.subplot(111, projection='3d')
axes.scatter(X.asnumpy(), Y.asnumpy(), Z.asnumpy(), 'r.')
net = gluon.nn.Sequential()
net.add(gluon.nn.Dense(200, activation='relu'))
net.add(gluon.nn.Dense(200, activation='relu'))
net.add(gluon.nn.Dense(1))
net.collect_params().initialize(mxnet.initializer.Xavier())
batch_size = 1000
dataset = gdata.ArrayDataset(XY, Z)
dataiter = gdata.DataLoader(dataset, batch_size=batch_size)
trainer = gluon.Trainer(net.collect_params(), 'adam', {
'beta1': .9,
'beta2': .999
})
def forward_multidims(self, inputs, loss):
results = [[] for i in range(self.multidims)]
for dim in range(self.multidims):
for i in range(self.slots):
results[dim].append(self.local_trans(inputs[i]))
results[dim].append(self.global_trans(inputs[-1]))
comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1))
if self.use_comm and self.topo_learning_mode:
proba = nd.sigmoid(self.topo.data())
if random.random() < 1e-2:
print '---------------------------------------------'
print proba.asnumpy()
print '---------------------------------------------'
u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1))
comm_rate = nd.sigmoid(10. * (
nd.log(proba) - nd.log(1. - proba) +
nd.log(u_vec) - nd.log(1. - u_vec)
))
if loss is not None:
loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba)))
f = [[None] * (self.slots + 1)] * (self.slots + 1)
if self.use_comm:
# local
for i in range(self.slots):
norm_fac = None
for j in range(self.slots):
if i != j:
f[i][j] = self.local_atten_out(self.local_atten_act(self.f_rec_local(inputs[i]) + self.f_emit_local2local(inputs[j])))
f[i][j] = nd.exp(f[i][j]).reshape((f[i][j].shape[0], self.multidims))
f[i][j] = f[i][j] * comm_rate[j][i]
if norm_fac is None:
norm_fac = nd.zeros_like(f[i][j])
norm_fac = norm_fac + f[i][j]
f[i][-1] = self.local_atten_out(self.local_atten_act(self.f_rec_local(inputs[i]) + self.f_emit_global2local(inputs[-1])))
# print '++++++++++++++++(*** x 3)++++++++++++++++++++++'
# print f[i][-1].shape
f[i][-1] = nd.exp(f[i][-1]).reshape((f[i][-1].shape[0], self.multidims))
f[i][-1] = f[i][-1] * comm_rate[-1][i]
if norm_fac is None:
norm_fac = nd.zeros_like(f[i][-1])
norm_fac = norm_fac + f[i][-1]
for j in range(self.slots):
if i != j:
f[i][j] = (1. / norm_fac) * f[i][j]
f[i][-1] = (1. / norm_fac) * f[i][-1]
for j in range(self.slots):
if i != j:
f[i][j] = mx.nd.split(f[i][j], axis=1, num_outputs=self.multidims)
f[i][-1] = mx.nd.split(f[i][-1], axis=1, num_outputs=self.multidims)
# print f[i][-1][0].shape
for dim in range(self.multidims):
for j in range(self.slots):
if i != j:
results[dim][i] = results[dim][i] + f[i][j][dim] * self.g_local2local(inputs[j])
results[dim][i] = results[dim][i] + f[i][-1][dim] * self.g_global2local(inputs[-1])
# global
norm_fac = None
for i in range(self.slots):
f[-1][i] = self.global_atten_out(self.global_atten_act(self.f_rec_global(inputs[-1]) + self.f_emit_local2global(inputs[i])))
f[-1][i] = nd.exp(f[-1][i]).reshape((f[-1][i].shape[0], self.multidims))
f[-1][i] = f[-1][i] * comm_rate[i][-1]
if norm_fac is None:
norm_fac = nd.zeros_like(f[-1][i])
norm_fac = norm_fac + f[-1][i]
for i in range(self.slots):
f[-1][i] = (1. / norm_fac) * f[-1][i]
f[-1][i] = mx.nd.split(f[-1][i], axis=1, num_outputs=self.multidims)
for dim in range(self.multidims):
for i in range(self.slots):
results[dim][-1] = results[dim][-1] + f[-1][i][dim] * self.g_local2global(inputs[i])
# norm = [None] * (self.slots + 1)
# for j in range(self.slots + 1):
# norm[j] = nd.zeros_like(f[j][0])
# for i in range(self.slots + 1):
# if i != j:
# norm[j] = norm[j] + f[j][i]
# for i in range(self.slots + 1):
# for j in range(self.slots + 1):
# if i == j:
# print nd.zeros_like(f[j][i]).asnumpy(),
# else:
# print (f[j][i] / norm[j]).asnumpy(),
# print ''
# print ''
multidims_add_results = []
for l in range(len(results[0])):
tmp = nd.zeros_like(results[0][l])
for dim in range(self.multidims):
tmp = tmp + results[dim][l]
multidims_add_results.append(tmp)
if self.block_mode:
assert self.local_in_units == self.local_units
assert self.global_in_units == self.global_units
for i in range(self.slots):
multidims_add_results[i] = self.yz_weight_local(multidims_add_results[i]) + inputs[i]
multidims_add_results[-1] = self.yz_weight_global(multidims_add_results[-1]) + inputs[-1]
return multidims_add_results
from mxnet import ndarray as nd
from mxnet.gluon import HybridBlock, nn
from mxnet.gluon.model_zoo import vision as models
class Net(HybridBlock):
def __init__(self, **kwargs):
super().__init__(**kwargs)
with self.name_scope():
self.features = models.resnet50_v2(pretrained=True).features
# self.features = models.densenet121(pretrained=True)
self.features.collect_params().setattr('lr_mult', 0.1)
self.output = nn.Dense(units=101)
def hybrid_forward(self, F, x, *args, **kwargs):
x = self.features(x)
x = self.output(x)
x = F.softmax(x)
return x
if __name__ == '__main__':
data = nd.random_uniform(shape=(4, 3, 224, 224))
net = Net()
net.collect_params().initialize()
output = net(data)
print(output)
c = nd.load(filename)
print(c)
# save & load gluon model parameters
def get_net():
net = gluon.nn.Sequential()
with net.name_scope():
net.add(gluon.nn.Dense(10, activation='relu'))
net.add(gluon.nn.Dense(2))
return net
net = get_net()
net.initialize()
x = nd.random_uniform(shape=(2, 10)) # 10个feature, 2 个样本
#net(x)
print(net(x))
print(x.shape)
print(net[0].weight.shape) # return 10 * 10
# save model
filename = './data/mpl.params'
net.save_params(filename)
# load model
net2 = get_net()
net2.load_params(filename, mx.cpu())
print(net2(x)) # same as net(x)
def __init__(self, **kwargs):
super(FancyMLP, self).__init__(**kwargs)
self.rand_weight = nd.random_uniform(shape=(10, 20))
with self.name_scope():
self.dense = gluon.nn.Dense(10, activation='relu')
def forward(self, inputs, loss=None):
assert len(inputs) == self.slots + 1
if self.non_local_mode:
return self.forward_multidims(inputs, loss)
if self.message_embedding:
return self.forward_message_embedding(inputs, loss)
local_drop_vec = nd.ones_like(inputs[0])
local_drop_vec = self.local_dropout_op(local_drop_vec)
for i in range(self.slots):
inputs[i] = inputs[i] * local_drop_vec
inputs[-1] = self.global_dropout_op(inputs[-1])
# local_share_vec = []
# local_private_vec = []
# if self.concrete_share_rate:
# raise ValueError('no share_private!!!')
# for i in range(self.slots):
# proba = nd.sigmoid(data=self.share_rate[i].data())
# proba = nd.broadcast_axis(data=proba, axis=(0, 1), size=inputs[0].shape)
# u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=inputs[0].shape, ctx=CTX)
# local_share_vec.append(nd.sigmoid(10. * (
# nd.log(proba) - nd.log(1. - proba) +
# nd.log(u_vec) - nd.log(1. - u_vec)
# )))
# local_private_vec.append(1. - local_share_vec[i])
# # print 'proba:', proba
# # print 'dropout_regularizer:', self.dropout_regularizer
# if loss is not None:
# loss.append(
# self.dropout_regularizer * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba)))
# if random.random() < 0.01:
# for i in range(self.slots):
# proba = nd.sigmoid(data=self.share_rate[i].data())
# print proba.asnumpy(),
# print ''
# else:
# local_share_vec = [nd.ones_like(inputs[0]), ] * self.slots
# local_private_vec = [nd.zeros_like(inputs[0]), ] * self.slots
# local_share_vec = (1. - self.private_rate) * nd.Dropout(
# nd.ones(shape=(inputs[0].shape[0], self.local_units)), p=self.private_rate, mode='always')
# local_private_vec = 1. - local_share_vec
comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1))
if self.use_comm and self.topo_learning_mode:
proba = nd.sigmoid(self.topo.data())
if random.random() < 1e-2:
print '---------------------------------------------'
print proba.asnumpy()
print '---------------------------------------------'
u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1))
comm_rate = nd.sigmoid(10. * (
nd.log(proba) - nd.log(1. - proba) +
nd.log(u_vec) - nd.log(1. - u_vec)
))
if loss is not None:
loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba)))
results = []
for i in range(self.slots):
results.append(self.local_share_trans(inputs[i]))
results.append(self.global_trans(inputs[-1]))
if self.use_comm:
if self.topo_learning_mode:
assert self.concrete_share_rate is False
for i in range(self.slots):
tmp = nd.zeros_like(results[i])
norm = nd.zeros_like(comm_rate[0][0])
for j in range(self.slots):
if i != j:
tmp = tmp + self.local2local_share_comm(inputs[j]) * comm_rate[j][i]
norm = norm + comm_rate[j][i]
# results[i] = results[i] + self.global2local_comm(inputs[-1]) * comm_rate[-1][i]
tmp = tmp + self.global2local_comm(inputs[-1]) * comm_rate[-1][i]
norm = norm + comm_rate[-1][i]
if nd.sum(norm) > 1e-5:
results[i] = results[i] + tmp / norm
tmp = nd.zeros_like(results[-1])
norm = nd.zeros_like(comm_rate[0][0])
for j in range(self.slots):
tmp = tmp + self.local2global_comm(inputs[j]) * comm_rate[j][-1]
norm = norm + comm_rate[j][-1]
if nd.sum(norm) > 1e-5:
results[-1] = results[-1] + tmp / norm
else:
for i in range(self.slots):
tmp = nd.zeros_like(results[i])
for j in range(self.slots):
if j != i:
tmp = tmp + self.local2local_share_comm(inputs[j])
tmp = tmp + self.global2local_comm(inputs[-1])
results[i] = results[i] + (tmp / float(self.slots))
tmp = nd.zeros_like(results[-1])
for i in range(self.slots):
tmp = tmp + self.local2global_comm(inputs[i])
results[-1] = results[-1] + (tmp / float(self.slots))
if self.block_mode:
assert self.local_in_units == self.local_units
assert self.global_in_units == self.global_units
for i in range(self.slots):
results[i] = self.yz_weight_local(results[i]) + inputs[i]
results[-1] = self.yz_weight_global(results[-1]) + inputs[-1]
return results
def forward(self, inputs, loss=None, training=True, commtype='average', topo='FC'):
assert len(inputs) == self.slots + 1
local_drop_vec = nd.ones_like(inputs[0])
local_drop_vec = self.local_dropout_op(local_drop_vec)
for i in range(self.slots):
inputs[i] = inputs[i] * local_drop_vec
inputs[-1] = self.global_dropout_op(inputs[-1])
if topo == 'FC':
comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1))
elif topo == 'FUC':
comm_rate = nd.zeros(shape=(self.slots + 1, self.slots + 1))
elif topo == 'Master':
comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1))
for i in range(self.slots):
for j in range(self.slots):
comm_rate[i][j] = 0
if self.use_comm and self.topo_learning_mode:
proba = nd.sigmoid(self.topo.data())
if random.random() < 1e-2:
print '---------------------------------------------'
print proba.asnumpy()
print '---------------------------------------------'
u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1))
comm_rate = nd.sigmoid(10. * (
nd.log(proba) - nd.log(1. - proba) +
nd.log(u_vec) - nd.log(1. - u_vec)
))
if loss is not None:
loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba)))
results = []
for i in range(self.slots):
results.append(self.local_share_trans.forward(inputs[i], training=training))
results.append(self.global_trans.forward(inputs[-1], training=training))
if commtype == 'average':
for i in range(self.slots):
tmp = nd.zeros_like(results[i])
norm = nd.zeros_like(comm_rate[0][0])
for j in range(self.slots):
if i != j:
tmp = tmp + self.local2local_share_comm.forward(nd.concat(inputs[j], dim=1),
training=training) * comm_rate[j][i]
norm = norm + comm_rate[j][i]
# results[i] = results[i] + self.global2local_comm(inputs[-1]) * comm_rate[-1][i]
tmp = tmp + self.global2local_comm.forward(nd.concat(inputs[-1], dim=1), training=training) * \
comm_rate[-1][i]
norm = norm + comm_rate[-1][i]
if nd.sum(norm) > 1e-5:
results[i] = results[i] + tmp / norm
tmp = nd.zeros_like(results[-1])
norm = nd.zeros_like(comm_rate[0][0])
for j in range(self.slots):
tmp = tmp + self.local2global_comm.forward(nd.concat(inputs[j], dim=1), training=training) * \
comm_rate[j][-1]
norm = norm + comm_rate[j][-1]
if nd.sum(norm) > 1e-5:
results[-1] = results[-1] + tmp / norm
elif commtype == 'maxpooling':
for i in range(self.slots):
tmp = []
for j in range(self.slots):
if j != i:
tmp.append(self.local2local_share_comm.forward(inputs[j], training=training))
tmp.append(self.global2local_comm.forward(inputs[-1], training=training))
for k in range(len(tmp)):
tmp[k] = tmp[k].reshape((tmp[k].shape[0], 1, tmp[k].shape[1]))
tmp = nd.concat(*tmp, dim=1)
maxcomm = nd.max(tmp, axis=1)
results[i] = results[i] + maxcomm
tmp = []
for i in range(self.slots):
tmp.append(self.local2global_comm.forward(inputs[i], training=training))
for k in range(len(tmp)):
tmp[k] = tmp[k].reshape((tmp[k].shape[0], 1, tmp[k].shape[1]))
tmp = nd.concat(*tmp, dim=1)
maxcomm = nd.max(tmp, axis=1)
results[-1] = results[-1] + maxcomm
return results
import cv2
def forward_non_local(self, inputs, loss, training):
results = []
for i in range(self.slots):
results.append(self.local_trans(inputs[i], training=training))
results.append(self.global_trans(inputs[-1], training=training))
comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1))
if self.use_comm and self.topo_learning_mode:
proba = nd.sigmoid(self.topo.data())
if random.random() < 1e-2:
print '---------------------------------------------'
print proba.asnumpy()
print '---------------------------------------------'
u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1))
comm_rate = nd.sigmoid(10. * (
nd.log(proba) - nd.log(1. - proba) +
nd.log(u_vec) - nd.log(1. - u_vec)
))
if loss is not None:
loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba)))
f = [[None] * (self.slots + 1)] * (self.slots + 1)
if self.use_comm:
# local
for i in range(self.slots):
norm_fac = None
for j in range(self.slots):
if i != j:
f[i][j] = nd.sum(self.f_rec_local(inputs[i], training=training) * self.f_emit_local2local(inputs[j], training=training), axis=1)
f[i][j] = nd.exp(f[i][j]).reshape((f[i][j].shape[0], 1))
f[i][j] = f[i][j] * comm_rate[j][i]
if norm_fac is None:
norm_fac = nd.zeros_like(f[i][j])
norm_fac = norm_fac + f[i][j]
f[i][-1] = nd.sum(self.f_rec_local(inputs[i], training=training) * self.f_emit_global2local(inputs[-1], training=training), axis=1)
f[i][-1] = nd.exp(f[i][-1]).reshape((f[i][-1].shape[0], 1))
f[i][-1] = f[i][-1] * comm_rate[-1][i]
if norm_fac is None:
norm_fac = nd.zeros_like(f[i][-1])
norm_fac = norm_fac + f[i][-1]
for j in range(self.slots):
if i != j:
results[i] = results[i] + (1. / norm_fac) * f[i][j] * self.g_local2local(inputs[j], training=training)
results[i] = results[i] + (1. / norm_fac) * f[i][-1] * self.g_global2local(inputs[-1], training=training)
# global
norm_fac = None
for i in range(self.slots):
f[-1][i] = nd.sum(self.f_rec_global(inputs[-1], training=training) * self.f_emit_local2global(inputs[i], training=training), axis=1)
f[-1][i] = nd.exp(f[-1][i]).reshape((f[-1][i].shape[0], 1))
f[-1][i] = f[-1][i] * comm_rate[i][-1]
if norm_fac is None:
norm_fac = nd.zeros_like(f[-1][i])
norm_fac = norm_fac + f[-1][i]
for i in range(self.slots):
results[-1] = results[-1] + (1. / norm_fac) * f[-1][i] * self.g_local2global(inputs[i], training=training)
if self.block_mode:
assert self.local_in_units == self.local_units
assert self.global_in_units == self.global_units
for i in range(self.slots):
results[i] = self.yz_weight_local(results[i], training=training) + inputs[i]
results[-1] = self.yz_weight_global(results[-1], training=training) + inputs[-1]
return results
def Noise(batch_size=None, ctx=None):
return nd.random_uniform(low=-1,
high=1,
shape=(batch_size, 100, 1, 1),
ctx=ctx)
