def __init__(self, vsize, ctx='cpu'): self.vsize = vsize self.ctx = mx.gpu() if ctx == 'gpu' else mx.cpu() self.W1 = nd.random_uniform(-.01, .01, shape=(vsize + 1024, 1024), ctx=self.ctx) self.b1 = nd.zeros(1024, ctx=self.ctx) self.W2 = nd.random_uniform(-.01, .01, shape=(2048, 1024), ctx=self.ctx) self.b2 = nd.zeros(1024, ctx=self.ctx) self.Wy = nd.random_uniform(-.01, .01, shape=(1024, vsize), ctx=self.ctx) self.by = nd.zeros(vsize, ctx=self.ctx) self.params = [self.W1, self.b1, self.W2, self.b2, self.Wy, self.by] for p in self.params: p.attach_grad()
def lkm_test(): net = LKM(pretrained=True) net.collect_params().initialize(init=mx.initializer.Xavier(magnitude=2.0), ctx=cfg.ctx) x = nd.random_uniform(shape=(2, 3, 512, 512)) y = net(x) print(y)
def deeplab_test(): net = DeepLab(pretrained=False) net.collect_params().initialize(init=mx.initializer.Xavier(magnitude=2.0), ctx=cfg.ctx) x = nd.random_uniform(shape=(2, 3, 448, 448)) y = net(x) print(y)
def TimeseriesFromPSD_nd(param_noise): """ GPU only """ (asd_pos, asd_neg, low_f, high_f, high_f_, size, fs, fmin, fmax) = param_noise (*D_, N) = size D = reduce(lambda x, y: x * y, D_) # Gauss noise and its one-sided PSD without window gauss_noise = 1* nd.random_normal(loc=0,scale=64,shape=(D, N), ctx=ctx) _, xf_noise, psd_gauss = oneSidedPeriodogram_nd(gauss_noise, fs=8192) psd_gauss = nd.array(psd_gauss, ctx = ctx, dtype='float64') psd_twosided = nd.concat( # low positive nd.zeros((D, low_f), ctx = ctx, dtype='float64'), # high positive psd_gauss[:, low_f:high_f] * asd_pos, nd.zeros((D, high_f_), ctx = ctx, dtype='float64'), nd.zeros((D, high_f_), ctx = ctx, dtype='float64'), # high negative psd_gauss[:, low_f:high_f][::-1] * asd_neg, # low negative nd.zeros((D, low_f), ctx = ctx, dtype='float64'), dim=1) amplitude = nd.sqrt(psd_twosided *2 *fs*N ) epsilon_imag = nd.random_uniform(low=0, high=1, shape=(D,N),ctx=ctx,dtype='float64')*2*np.pi re = nd.cos(epsilon_imag)*amplitude im = nd.sin(epsilon_imag)*amplitude temp = nd.zeros((D, N*2) , ctx=ctx) temp[:,::2] = re temp[:,1::2] = im timeseries = mx.contrib.ndarray.ifft(temp)/N return timeseries.reshape(size), psd_twosided
def __init__(self, n_vis=2, n_hid=1, n_val=2, ctx=mx.cpu()): self.n_vis = n_vis # num of units in visible (input) layer self.n_hid = n_hid # num of units in hidden layer self.n_val = n_val # num of values for each node self.n_node = n_vis * n_val # each visnode has n_val nodes self.ctx = ctx a = sqrt(6. / (self.n_vis * n_val + n_hid)) self.W = nd.random_uniform(low=-a, high=a, shape=(self.n_node, n_hid), ctx=self.ctx) self.hb = nd.zeros([n_hid], ctx=self.ctx) # initialize h bias 0 self.vb = nd.zeros([self.n_node], ctx=self.ctx) # initialize v bias 0 self.dW = nd.zeros([self.n_node, n_hid], ctx=self.ctx) self.dh = nd.zeros([n_hid], ctx=self.ctx) self.dv = nd.zeros([self.n_node], ctx=self.ctx) #for KL self.enum_states = None self.prob_states = None self.prob_RGs = None self.L_coeff = 0.0 self.M_coeff = 0.0 self.no_softmax = False
def shuffle_data_nd(data, peak_samppoint, peak_time, times): shift_list = nd.random_uniform(0, peak_samppoint - round((peak_time-0.2)*data.shape[-1]), shape=(times), ctx=mx.cpu()) base = forward_moving_wave_nd(data, int(shift_list.asnumpy()[0])) if times == 1: return base, shift_list else: for shift_size in shift_list[1:]: temp = forward_moving_wave_nd(data, int(shift_size.asnumpy()[0])) base = nd.concatenate([base, temp] , axis = 0) return base, shift_list
def query(self, images): if self.pool_size == 0: return images ret_imgs = [] for i in range(images.shape[0]): image = nd.expand_dims(images[i], axis=0) if self.num_imgs < self.pool_size: self.num_imgs = self.num_imgs + 1 self.images.append(image) ret_imgs.append(image) else: p = nd.random_uniform(0, 1, shape=(1,)).asscalar() if p > 0.5: random_id = nd.random_uniform(0, self.pool_size - 1, shape=(1,)).astype(np.uint8).asscalar() tmp = self.images[random_id].copy() self.images[random_id] = image ret_imgs.append(tmp) else: ret_imgs.append(image) ret_imgs = nd.concat(*ret_imgs, dim=0) return ret_imgs
def forward(self, is_train, req, in_data, out_data, aux): x = in_data[0] if is_train: self._spatial_dropout_mask = F.broadcast_greater( F.random_uniform(low=0, high=1, shape=(1, self._num_filters, 1, 1), ctx=self._ctx), F.ones(shape=(1, self._num_filters, 1, 1), ctx=self._ctx) * self._p, ctx=self._ctx ) y = F.broadcast_mul(x, self._spatial_dropout_mask, ctx=self._ctx) / (1-self._p) self.assign(out_data[0], req[0], y) else: self.assign(out_data[0], req[0], x)
def dropout(batch_X, drop_probability): keep_probability = 1 - drop_probability assert 0 <= keep_probability <= 1 if keep_probability == 0: return batch_X.zeros_like() # > 保存的概率才能够保留该样本该神经元的输出 mask = nd.random_uniform( 0, 1.0, batch_X.shape, ctx=batch_X.context) < keep_probability # 保证 E[dropout(batch_X)] == batch_X scale = 1 / keep_probability return mask * batch_X * scale
def __init__(self, vsize, ctx='cpu'): self.vsize = vsize self.ctx = mx.gpu() if ctx == 'gpu' else mx.cpu() self.a1 = init_gru_args(vsize, 1024, self.ctx) self.Wy = nd.random_uniform(-.01, .01, shape=(1024, vsize), ctx=self.ctx) self.by = nd.zeros(vsize, ctx=self.ctx) self.params = [*self.a1, self.Wy, self.by] for p in self.params: p.attach_grad()
def init_lstm_args(xsize, hsize, ctx): return ( nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(xsize, hsize), ctx=ctx), nd.random_uniform(-.01, .01, shape=(hsize, hsize), ctx=ctx), nd.zeros(hsize, ctx=ctx), nd.zeros(hsize, ctx=ctx), nd.zeros(hsize, ctx=ctx), nd.zeros(hsize, ctx=ctx) )
def test_large_models(): ctx = default_context() # Create model net = gluon.nn.HybridSequential() largest_num_features = 256 net.add(nn.Conv2D(largest_num_features, 3)) net.hybridize() net.initialize(mx.init.Normal(sigma=0.01), ctx=ctx) # Compute the height (=width) of the square tensor of the given size in bytes def tensor_size(big_tensor_bytes): bytes_per_float = 4 sz = int( math.sqrt(big_tensor_bytes / largest_num_features / bytes_per_float)) return (sz // 100) * 100 # The idea is to create models with large tensors of (say) 20% of the total memory. # This in the past has given cudnnFind() trouble when it needed to allocate similar I/O's # from the area carved out by the MXNET_GPU_MEM_POOL_RESERVE setting (by default 5%). (free_mem_bytes, total_mem_bytes) = mx.context.gpu_memory_info(ctx.device_id) # This test needs to be 'qualified' for use with each new larger memory size largest_supported_total_mem_GB = 32 if (total_mem_bytes > largest_supported_total_mem_GB * 1024 * 1024 * 1024): sys.stderr.write( ' bypassing test due to too-large global memory of size {} ... '. format(total_mem_bytes)) return start_size = tensor_size(0.20 * total_mem_bytes) num_trials = 10 sys.stderr.write( ' testing global memory of size {} ... '.format(total_mem_bytes)) sys.stderr.flush() for i in range(num_trials): sz = start_size - 10 * i (height, width) = (sz, sz) sys.stderr.write(" {}x{} ".format(height, width)) sys.stderr.flush() data_in = nd.random_uniform(low=0, high=255, shape=(1, 3, height, width), ctx=ctx, dtype="float32") # Evaluate model net(data_in).asnumpy()
def daydream(self, n, k=1): sys.stderr.write("Day dreaming ...\n") h0 = nd.random_uniform(low=0, high=1, shape=[1, self.n_hid], ctx=self.ctx) for i in range(100): v_prob, v = self.sample_v_given_h(h0) h_prob, h0 = self.sample_h_given_v(v) for i in range(n): for cdk in range(k): v_prob, v = self.sample_v_given_h(h0) h_prob, h0 = self.sample_h_given_v(v) ndx = nd.argmax(v.reshape([-1, self.n_val]), axis=1) for n in ndx.asnumpy(): sys.stdout.write(str(int(n))) sys.stdout.write("\n") return
def sample_v_given_h_without_softmax(self, h0): v1_prob = nd.sigmoid(self.propdown(h0)) v1 = v1_prob > nd.random_uniform( low=0.0, high=1.0, shape=v1_prob.shape, ctx=self.ctx) return [v1_prob, v1]
def sample_h_given_v(self, v0): h1_prob = self.propup(v0) h1 = h1_prob > nd.random_uniform( low=0.0, high=1.0, shape=h1_prob.shape, ctx=self.ctx) return [h1_prob, h1]
def __init__(self, **kwargs): super(FancyBlock, self).__init__(**kwargs) with self.name_scope(): self.dense = gluon.nn.Dense(256) self.weight = nd.random_uniform(shape=(256, 20))
from mxnet import gluon from mxnet import ndarray as nd from mxnet import autograd from mxnet.gluon import data as gdata from mpl_toolkits.mplot3d import Axes3D from matplotlib import pyplot as plt import mxnet losser = gluon.loss.L2Loss() X = nd.random_uniform(-100, 100, shape=(1000)) Y = nd.random_uniform(-100, 100, shape=(1000)) XY = nd.concat(X.reshape((-1, 1)), Y.reshape((-1, 1)), dim=1) Z = nd.exp(-nd.abs(X - Y)) figure = plt.figure() axes = plt.subplot(111, projection='3d') axes.scatter(X.asnumpy(), Y.asnumpy(), Z.asnumpy(), 'r.') net = gluon.nn.Sequential() net.add(gluon.nn.Dense(200, activation='relu')) net.add(gluon.nn.Dense(200, activation='relu')) net.add(gluon.nn.Dense(1)) net.collect_params().initialize(mxnet.initializer.Xavier()) batch_size = 1000 dataset = gdata.ArrayDataset(XY, Z) dataiter = gdata.DataLoader(dataset, batch_size=batch_size) trainer = gluon.Trainer(net.collect_params(), 'adam', { 'beta1': .9, 'beta2': .999 })
def forward_multidims(self, inputs, loss): results = [[] for i in range(self.multidims)] for dim in range(self.multidims): for i in range(self.slots): results[dim].append(self.local_trans(inputs[i])) results[dim].append(self.global_trans(inputs[-1])) comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1)) if self.use_comm and self.topo_learning_mode: proba = nd.sigmoid(self.topo.data()) if random.random() < 1e-2: print '---------------------------------------------' print proba.asnumpy() print '---------------------------------------------' u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1)) comm_rate = nd.sigmoid(10. * ( nd.log(proba) - nd.log(1. - proba) + nd.log(u_vec) - nd.log(1. - u_vec) )) if loss is not None: loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba))) f = [[None] * (self.slots + 1)] * (self.slots + 1) if self.use_comm: # local for i in range(self.slots): norm_fac = None for j in range(self.slots): if i != j: f[i][j] = self.local_atten_out(self.local_atten_act(self.f_rec_local(inputs[i]) + self.f_emit_local2local(inputs[j]))) f[i][j] = nd.exp(f[i][j]).reshape((f[i][j].shape[0], self.multidims)) f[i][j] = f[i][j] * comm_rate[j][i] if norm_fac is None: norm_fac = nd.zeros_like(f[i][j]) norm_fac = norm_fac + f[i][j] f[i][-1] = self.local_atten_out(self.local_atten_act(self.f_rec_local(inputs[i]) + self.f_emit_global2local(inputs[-1]))) # print '++++++++++++++++(*** x 3)++++++++++++++++++++++' # print f[i][-1].shape f[i][-1] = nd.exp(f[i][-1]).reshape((f[i][-1].shape[0], self.multidims)) f[i][-1] = f[i][-1] * comm_rate[-1][i] if norm_fac is None: norm_fac = nd.zeros_like(f[i][-1]) norm_fac = norm_fac + f[i][-1] for j in range(self.slots): if i != j: f[i][j] = (1. / norm_fac) * f[i][j] f[i][-1] = (1. / norm_fac) * f[i][-1] for j in range(self.slots): if i != j: f[i][j] = mx.nd.split(f[i][j], axis=1, num_outputs=self.multidims) f[i][-1] = mx.nd.split(f[i][-1], axis=1, num_outputs=self.multidims) # print f[i][-1][0].shape for dim in range(self.multidims): for j in range(self.slots): if i != j: results[dim][i] = results[dim][i] + f[i][j][dim] * self.g_local2local(inputs[j]) results[dim][i] = results[dim][i] + f[i][-1][dim] * self.g_global2local(inputs[-1]) # global norm_fac = None for i in range(self.slots): f[-1][i] = self.global_atten_out(self.global_atten_act(self.f_rec_global(inputs[-1]) + self.f_emit_local2global(inputs[i]))) f[-1][i] = nd.exp(f[-1][i]).reshape((f[-1][i].shape[0], self.multidims)) f[-1][i] = f[-1][i] * comm_rate[i][-1] if norm_fac is None: norm_fac = nd.zeros_like(f[-1][i]) norm_fac = norm_fac + f[-1][i] for i in range(self.slots): f[-1][i] = (1. / norm_fac) * f[-1][i] f[-1][i] = mx.nd.split(f[-1][i], axis=1, num_outputs=self.multidims) for dim in range(self.multidims): for i in range(self.slots): results[dim][-1] = results[dim][-1] + f[-1][i][dim] * self.g_local2global(inputs[i]) # norm = [None] * (self.slots + 1) # for j in range(self.slots + 1): # norm[j] = nd.zeros_like(f[j][0]) # for i in range(self.slots + 1): # if i != j: # norm[j] = norm[j] + f[j][i] # for i in range(self.slots + 1): # for j in range(self.slots + 1): # if i == j: # print nd.zeros_like(f[j][i]).asnumpy(), # else: # print (f[j][i] / norm[j]).asnumpy(), # print '' # print '' multidims_add_results = [] for l in range(len(results[0])): tmp = nd.zeros_like(results[0][l]) for dim in range(self.multidims): tmp = tmp + results[dim][l] multidims_add_results.append(tmp) if self.block_mode: assert self.local_in_units == self.local_units assert self.global_in_units == self.global_units for i in range(self.slots): multidims_add_results[i] = self.yz_weight_local(multidims_add_results[i]) + inputs[i] multidims_add_results[-1] = self.yz_weight_global(multidims_add_results[-1]) + inputs[-1] return multidims_add_results
from mxnet import ndarray as nd from mxnet.gluon import HybridBlock, nn from mxnet.gluon.model_zoo import vision as models class Net(HybridBlock): def __init__(self, **kwargs): super().__init__(**kwargs) with self.name_scope(): self.features = models.resnet50_v2(pretrained=True).features # self.features = models.densenet121(pretrained=True) self.features.collect_params().setattr('lr_mult', 0.1) self.output = nn.Dense(units=101) def hybrid_forward(self, F, x, *args, **kwargs): x = self.features(x) x = self.output(x) x = F.softmax(x) return x if __name__ == '__main__': data = nd.random_uniform(shape=(4, 3, 224, 224)) net = Net() net.collect_params().initialize() output = net(data) print(output)
c = nd.load(filename) print(c) # save & load gluon model parameters def get_net(): net = gluon.nn.Sequential() with net.name_scope(): net.add(gluon.nn.Dense(10, activation='relu')) net.add(gluon.nn.Dense(2)) return net net = get_net() net.initialize() x = nd.random_uniform(shape=(2, 10)) # 10个feature, 2 个样本 #net(x) print(net(x)) print(x.shape) print(net[0].weight.shape) # return 10 * 10 # save model filename = './data/mpl.params' net.save_params(filename) # load model net2 = get_net() net2.load_params(filename, mx.cpu()) print(net2(x)) # same as net(x)
def __init__(self, **kwargs): super(FancyMLP, self).__init__(**kwargs) self.rand_weight = nd.random_uniform(shape=(10, 20)) with self.name_scope(): self.dense = gluon.nn.Dense(10, activation='relu')
def forward(self, inputs, loss=None): assert len(inputs) == self.slots + 1 if self.non_local_mode: return self.forward_multidims(inputs, loss) if self.message_embedding: return self.forward_message_embedding(inputs, loss) local_drop_vec = nd.ones_like(inputs[0]) local_drop_vec = self.local_dropout_op(local_drop_vec) for i in range(self.slots): inputs[i] = inputs[i] * local_drop_vec inputs[-1] = self.global_dropout_op(inputs[-1]) # local_share_vec = [] # local_private_vec = [] # if self.concrete_share_rate: # raise ValueError('no share_private!!!') # for i in range(self.slots): # proba = nd.sigmoid(data=self.share_rate[i].data()) # proba = nd.broadcast_axis(data=proba, axis=(0, 1), size=inputs[0].shape) # u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=inputs[0].shape, ctx=CTX) # local_share_vec.append(nd.sigmoid(10. * ( # nd.log(proba) - nd.log(1. - proba) + # nd.log(u_vec) - nd.log(1. - u_vec) # ))) # local_private_vec.append(1. - local_share_vec[i]) # # print 'proba:', proba # # print 'dropout_regularizer:', self.dropout_regularizer # if loss is not None: # loss.append( # self.dropout_regularizer * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba))) # if random.random() < 0.01: # for i in range(self.slots): # proba = nd.sigmoid(data=self.share_rate[i].data()) # print proba.asnumpy(), # print '' # else: # local_share_vec = [nd.ones_like(inputs[0]), ] * self.slots # local_private_vec = [nd.zeros_like(inputs[0]), ] * self.slots # local_share_vec = (1. - self.private_rate) * nd.Dropout( # nd.ones(shape=(inputs[0].shape[0], self.local_units)), p=self.private_rate, mode='always') # local_private_vec = 1. - local_share_vec comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1)) if self.use_comm and self.topo_learning_mode: proba = nd.sigmoid(self.topo.data()) if random.random() < 1e-2: print '---------------------------------------------' print proba.asnumpy() print '---------------------------------------------' u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1)) comm_rate = nd.sigmoid(10. * ( nd.log(proba) - nd.log(1. - proba) + nd.log(u_vec) - nd.log(1. - u_vec) )) if loss is not None: loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba))) results = [] for i in range(self.slots): results.append(self.local_share_trans(inputs[i])) results.append(self.global_trans(inputs[-1])) if self.use_comm: if self.topo_learning_mode: assert self.concrete_share_rate is False for i in range(self.slots): tmp = nd.zeros_like(results[i]) norm = nd.zeros_like(comm_rate[0][0]) for j in range(self.slots): if i != j: tmp = tmp + self.local2local_share_comm(inputs[j]) * comm_rate[j][i] norm = norm + comm_rate[j][i] # results[i] = results[i] + self.global2local_comm(inputs[-1]) * comm_rate[-1][i] tmp = tmp + self.global2local_comm(inputs[-1]) * comm_rate[-1][i] norm = norm + comm_rate[-1][i] if nd.sum(norm) > 1e-5: results[i] = results[i] + tmp / norm tmp = nd.zeros_like(results[-1]) norm = nd.zeros_like(comm_rate[0][0]) for j in range(self.slots): tmp = tmp + self.local2global_comm(inputs[j]) * comm_rate[j][-1] norm = norm + comm_rate[j][-1] if nd.sum(norm) > 1e-5: results[-1] = results[-1] + tmp / norm else: for i in range(self.slots): tmp = nd.zeros_like(results[i]) for j in range(self.slots): if j != i: tmp = tmp + self.local2local_share_comm(inputs[j]) tmp = tmp + self.global2local_comm(inputs[-1]) results[i] = results[i] + (tmp / float(self.slots)) tmp = nd.zeros_like(results[-1]) for i in range(self.slots): tmp = tmp + self.local2global_comm(inputs[i]) results[-1] = results[-1] + (tmp / float(self.slots)) if self.block_mode: assert self.local_in_units == self.local_units assert self.global_in_units == self.global_units for i in range(self.slots): results[i] = self.yz_weight_local(results[i]) + inputs[i] results[-1] = self.yz_weight_global(results[-1]) + inputs[-1] return results
def forward(self, inputs, loss=None, training=True, commtype='average', topo='FC'): assert len(inputs) == self.slots + 1 local_drop_vec = nd.ones_like(inputs[0]) local_drop_vec = self.local_dropout_op(local_drop_vec) for i in range(self.slots): inputs[i] = inputs[i] * local_drop_vec inputs[-1] = self.global_dropout_op(inputs[-1]) if topo == 'FC': comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1)) elif topo == 'FUC': comm_rate = nd.zeros(shape=(self.slots + 1, self.slots + 1)) elif topo == 'Master': comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1)) for i in range(self.slots): for j in range(self.slots): comm_rate[i][j] = 0 if self.use_comm and self.topo_learning_mode: proba = nd.sigmoid(self.topo.data()) if random.random() < 1e-2: print '---------------------------------------------' print proba.asnumpy() print '---------------------------------------------' u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1)) comm_rate = nd.sigmoid(10. * ( nd.log(proba) - nd.log(1. - proba) + nd.log(u_vec) - nd.log(1. - u_vec) )) if loss is not None: loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba))) results = [] for i in range(self.slots): results.append(self.local_share_trans.forward(inputs[i], training=training)) results.append(self.global_trans.forward(inputs[-1], training=training)) if commtype == 'average': for i in range(self.slots): tmp = nd.zeros_like(results[i]) norm = nd.zeros_like(comm_rate[0][0]) for j in range(self.slots): if i != j: tmp = tmp + self.local2local_share_comm.forward(nd.concat(inputs[j], dim=1), training=training) * comm_rate[j][i] norm = norm + comm_rate[j][i] # results[i] = results[i] + self.global2local_comm(inputs[-1]) * comm_rate[-1][i] tmp = tmp + self.global2local_comm.forward(nd.concat(inputs[-1], dim=1), training=training) * \ comm_rate[-1][i] norm = norm + comm_rate[-1][i] if nd.sum(norm) > 1e-5: results[i] = results[i] + tmp / norm tmp = nd.zeros_like(results[-1]) norm = nd.zeros_like(comm_rate[0][0]) for j in range(self.slots): tmp = tmp + self.local2global_comm.forward(nd.concat(inputs[j], dim=1), training=training) * \ comm_rate[j][-1] norm = norm + comm_rate[j][-1] if nd.sum(norm) > 1e-5: results[-1] = results[-1] + tmp / norm elif commtype == 'maxpooling': for i in range(self.slots): tmp = [] for j in range(self.slots): if j != i: tmp.append(self.local2local_share_comm.forward(inputs[j], training=training)) tmp.append(self.global2local_comm.forward(inputs[-1], training=training)) for k in range(len(tmp)): tmp[k] = tmp[k].reshape((tmp[k].shape[0], 1, tmp[k].shape[1])) tmp = nd.concat(*tmp, dim=1) maxcomm = nd.max(tmp, axis=1) results[i] = results[i] + maxcomm tmp = [] for i in range(self.slots): tmp.append(self.local2global_comm.forward(inputs[i], training=training)) for k in range(len(tmp)): tmp[k] = tmp[k].reshape((tmp[k].shape[0], 1, tmp[k].shape[1])) tmp = nd.concat(*tmp, dim=1) maxcomm = nd.max(tmp, axis=1) results[-1] = results[-1] + maxcomm return results
import cv2
def forward_non_local(self, inputs, loss, training): results = [] for i in range(self.slots): results.append(self.local_trans(inputs[i], training=training)) results.append(self.global_trans(inputs[-1], training=training)) comm_rate = nd.ones(shape=(self.slots + 1, self.slots + 1)) if self.use_comm and self.topo_learning_mode: proba = nd.sigmoid(self.topo.data()) if random.random() < 1e-2: print '---------------------------------------------' print proba.asnumpy() print '---------------------------------------------' u_vec = nd.random_uniform(low=1e-5, high=1. - 1e-5, shape=(self.slots + 1, self.slots + 1)) comm_rate = nd.sigmoid(10. * ( nd.log(proba) - nd.log(1. - proba) + nd.log(u_vec) - nd.log(1. - u_vec) )) if loss is not None: loss.append(4e-4 * nd.sum(proba * nd.log(proba) + (1. - proba) * nd.log(1. - proba))) f = [[None] * (self.slots + 1)] * (self.slots + 1) if self.use_comm: # local for i in range(self.slots): norm_fac = None for j in range(self.slots): if i != j: f[i][j] = nd.sum(self.f_rec_local(inputs[i], training=training) * self.f_emit_local2local(inputs[j], training=training), axis=1) f[i][j] = nd.exp(f[i][j]).reshape((f[i][j].shape[0], 1)) f[i][j] = f[i][j] * comm_rate[j][i] if norm_fac is None: norm_fac = nd.zeros_like(f[i][j]) norm_fac = norm_fac + f[i][j] f[i][-1] = nd.sum(self.f_rec_local(inputs[i], training=training) * self.f_emit_global2local(inputs[-1], training=training), axis=1) f[i][-1] = nd.exp(f[i][-1]).reshape((f[i][-1].shape[0], 1)) f[i][-1] = f[i][-1] * comm_rate[-1][i] if norm_fac is None: norm_fac = nd.zeros_like(f[i][-1]) norm_fac = norm_fac + f[i][-1] for j in range(self.slots): if i != j: results[i] = results[i] + (1. / norm_fac) * f[i][j] * self.g_local2local(inputs[j], training=training) results[i] = results[i] + (1. / norm_fac) * f[i][-1] * self.g_global2local(inputs[-1], training=training) # global norm_fac = None for i in range(self.slots): f[-1][i] = nd.sum(self.f_rec_global(inputs[-1], training=training) * self.f_emit_local2global(inputs[i], training=training), axis=1) f[-1][i] = nd.exp(f[-1][i]).reshape((f[-1][i].shape[0], 1)) f[-1][i] = f[-1][i] * comm_rate[i][-1] if norm_fac is None: norm_fac = nd.zeros_like(f[-1][i]) norm_fac = norm_fac + f[-1][i] for i in range(self.slots): results[-1] = results[-1] + (1. / norm_fac) * f[-1][i] * self.g_local2global(inputs[i], training=training) if self.block_mode: assert self.local_in_units == self.local_units assert self.global_in_units == self.global_units for i in range(self.slots): results[i] = self.yz_weight_local(results[i], training=training) + inputs[i] results[-1] = self.yz_weight_global(results[-1], training=training) + inputs[-1] return results
def Noise(batch_size=None, ctx=None): return nd.random_uniform(low=-1, high=1, shape=(batch_size, 100, 1, 1), ctx=ctx)