def maybe_init(self, in_channels): if self.init: return elif in_channels is None: return if DEBUG1: print('maybe_init', in_channels) pad = self.ksize // 2 if self.padding else 0 with self.init_scope(): self.enc = L.Convolution2D(in_channels, self.out_channels, ksize=self.ksize, stride=1, pad=pad) self.dec = L.Deconvolution2D(self.out_channels, in_channels, ksize=self.ksize, stride=1, pad=pad) if self.batch_norm: self.bne = L.BatchRenormalization(self.out_channels) self.bnd = L.BatchRenormalization(in_channels) else: # self.bne = L.BatchNormalization(self.out_size) # self.bnd = L.BatchNormalization(in_size) self.bne = None self.bnd = None self.in_channels = in_channels self.init = True self.adjust()
def maybe_init(self, in_size_): if self.init: return elif in_size_ is None: return if type(in_size_) is tuple: in_size = np.prod(in_size_) if DEBUG1: print('maybe_init', in_size_, '->', in_size) else: in_size = in_size_ if DEBUG1: print('maybe_init', in_size) with self.init_scope(): self.enc = L.Linear(in_size, self.out_size) self.dec = L.Linear(self.out_size, in_size) if self.batch_norm: self.bne = L.BatchRenormalization(self.out_size) self.bnd = L.BatchRenormalization(in_size) else: # self.bne = L.BatchNormalization(self.out_size) # self.bnd = L.BatchNormalization(in_size) self.bne = None self.bnd = None self.in_size = in_size self.init = True self.adjust(device=self.device)
def __init__(self, channels, ksize, stride, pad): super(ResBlock, self).__init__() with self.init_scope(): self.conv1 = L.Convolution2D(channels, channels, ksize, stride, pad) self.norm1 = L.BatchRenormalization(channels) self.conv2 = L.Convolution2D(channels, channels, ksize, stride, pad) self.norm2 = L.BatchRenormalization(channels) self.conv3 = L.Convolution2D(channels, channels, ksize, stride, pad) self.norm3 = L.BatchRenormalization(channels)
def setUp(self): self.decay = 0.9 self.size = 3 self.link = links.BatchRenormalization(self.size, self.decay) self.x = numpy.random.uniform(-1, 1, (self.nx, self.size)).astype( numpy.float32) self.y = numpy.random.uniform(-1, 1, (self.ny, self.size)).astype( numpy.float32)
def __init__(self, link_name, finetune, forget=False): super(Model, self).__init__() with self.init_scope(): if link_name == 'bn': self.link = links.BatchNormalization(3) elif link_name == 'brn': self.link = links.BatchRenormalization(3) self.forget = forget self.finetune = finetune
def maybe_init(self, in_channels): if self.init: return elif in_channels is None: return if DEBUG1: print('maybe_init', in_channels) pad = self.ksize // 2 if self.padding else 0 def mk_enc(i): ch_in = self.out_channels if i else in_channels ch_out = self.out_channels return L.Convolution2D(ch_in, ch_out, ksize=self.ksize, pad=pad) def mk_dec(i): ch_in = self.out_channels ch_out = self.out_channels if i else in_channels return L.Deconvolution2D(ch_in, ch_out, ksize=self.ksize, pad=pad) enc_s = list(map(mk_enc, range(self.n_conv))) dec_s = list(map(mk_dec, reversed(range(self.n_conv)))) self.enc = lambda x: reduce(lambda h, f: f(h), enc_s, x) self.dec = lambda x: reduce(lambda h, f: f(h), dec_s, x) for i, f in enumerate(enc_s): self.add_link(f'enc{i}', f) # =~ setattr(self, 'enc*', f) for i, f in enumerate(dec_s): self.add_link(f'dec{i}', f) # =~ setattr(self, 'dec*', f) with self.init_scope(): if self.batch_norm: self.bne = L.BatchRenormalization(self.out_channels) self.bnd = L.BatchRenormalization(in_channels) else: self.bne = None self.bnd = None self.in_channels = in_channels self.init = True self.adjust()
def __init__(self): super(VGG_ReNorm, self).__init__() with self.init_scope(): # Convolution self.conv1_1 = L.Convolution2D(None, 64, (3, 3), stride=1, pad=1) #64->64 self.conv1_2 = L.Convolution2D(None, 64, (3, 3), stride=1, pad=1) #64->64 self.conv2_1 = L.Convolution2D(None, 128, (3, 3), stride=1, pad=1) #32->32 self.conv2_2 = L.Convolution2D(None, 128, (3, 3), stride=1, pad=1) #32->32 self.conv3_1 = L.Convolution2D(None, 256, (3, 3), stride=1, pad=1) #16->16 self.conv3_2 = L.Convolution2D(None, 256, (3, 3), stride=1, pad=1) #16->16 self.conv3_3 = L.Convolution2D(None, 256, (3, 3), stride=1, pad=1) #16->16 self.conv4_1 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #8->8 self.conv4_2 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #8->8 self.conv4_3 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #8->8 self.conv5_1 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #4->4 self.conv5_2 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #4->4 self.conv5_3 = L.Convolution2D(None, 512, (3, 3), stride=1, pad=1) #4->4 # BatchNormalize self.bn1 = L.BatchRenormalization(64) self.bn2 = L.BatchRenormalization(128) self.bn3 = L.BatchRenormalization(256) self.bn4 = L.BatchRenormalization(512) self.bn5 = L.BatchRenormalization(512) self.fc6 = L.Linear(None, 4096) self.fc7 = L.Linear(None, 4096) self.fc8 = L.Linear(None, 2)
def __init__(self, channel, ksize, dilate=1, dropout_rate = 0.3, causal=True): self.dropout_rate = dropout_rate self.half = channel ls = {} ls["c"] = DilateConvCausal1D(channel, channel * 2, ksize, dilate=dilate, causal=causal) ls["bn"] = L.BatchRenormalization(channel*2, decay=0.9, eps=2e-5) super(GatedConv1D, self).__init__(**ls)
def setUp(self): self.expander = (None, Ellipsis) + (None, ) * self.ndim self.aggr_axes = (0, ) + tuple(six.moves.range(2, self.ndim + 2)) self.rmax = self.dtype(3) self.dmax = self.dtype(5) self.link = links.BatchRenormalization(3, rmax=self.rmax, dmax=self.dmax, dtype=self.dtype) gamma = self.link.gamma.data gamma[...] = numpy.random.uniform(.5, 1, gamma.shape) beta = self.link.beta.data beta[...] = numpy.random.uniform(-1, 1, beta.shape) self.link.cleargrads() self.gamma = gamma.copy()[self.expander] # fixed on CPU self.beta = beta.copy()[self.expander] # fixed on CPU shape = (5, 3) + (2, ) * self.ndim self.x = numpy.random.uniform(-1, 1, shape).astype(self.dtype) self.gy = numpy.random.uniform(-1, 1, shape).astype(self.dtype) if self.test: self.mean = numpy.random.uniform(-1, 1, (3, )).astype(self.dtype) self.var = numpy.random.uniform(0.5, 1, (3, )).astype(self.dtype) self.link.avg_mean[...] = self.mean self.link.avg_var[...] = self.var self.running_mean = self.mean self.running_var = self.var else: self.mean = self.x.mean(axis=self.aggr_axes) self.var = self.x.var(axis=self.aggr_axes) # Need to add some noise to running_mean and running_var, # otherwise we will always get r=1, d=0 self.running_mean = self.mean + numpy.random.uniform( -1, 1, self.mean.shape).astype(self.dtype) self.running_var = numpy.abs( self.var + numpy.random.uniform(-1, 1, self.var.shape).astype(self.dtype)) self.link.avg_mean[...] = self.running_mean self.link.avg_var[...] = self.running_var self.check_forward_optionss = {'atol': 1e-4, 'rtol': 1e-3} self.check_backward_optionss = {'atol': 1e-4, 'rtol': 1e-3} if self.dtype == numpy.float16: self.check_forward_optionss = {'atol': 1e-3, 'rtol': 1e-2} self.check_backward_optionss = {'atol': 5e-1, 'rtol': 1e-1}
def __init__(self, link_name, finetune, forget=False): super(Model, self).__init__() with self.init_scope(): if link_name == 'bn': self.link = links.BatchNormalization(3) elif link_name == 'brn': # defaults rmax=1, dmax=0 are so trivial that BRN # becomes BN self.link = links.BatchRenormalization(3, rmax=2.0, dmax=1.0) elif link_name == 'dbn': self.link = links.DecorrelatedBatchNormalization( 3, groups=3) self.forget = forget self.finetune = finetune
def __init__(self, n_unit, ksize, stride, pad, actfun=None, dropout=0, wd=0.02): super(DownSampleBlock, self).__init__() with self.init_scope(): self.cnv = L.Convolution2D(None, n_unit, ksize=ksize, stride=stride, pad=pad, initialW=I.Normal(wd)) self.brn = L.BatchRenormalization(n_unit) self.actfun = actfun self.dropout_ratio = dropout