def _lstm_cell(self, name, n_hidden, x_in, h=None, c=None): if h is None: h = nn.Variable.from_numpy_array( np.zeros((self._batch_size, self._cols_size))) if c is None: c = nn.Variable.from_numpy_array( np.zeros((self._batch_size, n_hidden))) h = F.concatenate(h, x_in, axis=1) # LSTM_Concatenate -> cols_size * 2 with nn.parameter_scope(name + '_Affine'): # LSTM_Affine -> n_hidden h1 = PF.affine(h, (n_hidden, ), base_axis=1) with nn.parameter_scope(name + '_IGate'): # LSTM_IGate -> n_hidden h2 = PF.affine(h, (n_hidden, ), base_axis=1) with nn.parameter_scope(name + '_FGate'): # LSTM_FGate -> n_hidden h3 = PF.affine(h, (n_hidden, ), base_axis=1) with nn.parameter_scope(name + '_OGate'): # LSTM_OGate -> n_hidden h4 = PF.affine(h, (n_hidden, ), base_axis=1) h1 = F.tanh(h1) # LSTM_Tanh h2 = F.sigmoid(h2) # LSTM_Sigmoid h3 = F.sigmoid(h3) # LSTM_Sigmoid_2 h4 = F.sigmoid(h4) # LSTM_Sigmoid_3 h5 = F.mul2(h2, h1) # LSTM_Mul2 -> n_hidden h6 = F.mul2(h3, c) # LSTM_Mul2_2 -> n_hidden h7 = F.add2(h5, h6, inplace=True) # LSTM_Add2 -> n_hidden h8 = F.tanh(h7) # LSTM_Tanh_2 -> n_hidden h9 = F.mul2(h4, h8) # LSTM_Mul2_3 -> n_hidden c = h7 # LSTM_C h = h9 # LSTM_H return (h, c)
def cnn(x): """Unnecessarily Deep CNN. Args: x : Variable, shape (B, 1, 8, 8) Returns: y : Variable, shape (B, 10) """ with nn.parameter_scope("cnn"): # Parameter scope can be nested with nn.parameter_scope("conv1"): h = F.tanh( PF.batch_normalization(PF.convolution(x, 64, (3, 3), pad=(1, 1))) ) for i in range(10): # unnecessarily deep with nn.parameter_scope("conv{}".format(i + 2)): h = F.tanh( PF.batch_normalization(PF.convolution(h, 128, (3, 3), pad=(1, 1))) ) with nn.parameter_scope("conv_last"): h = F.tanh( PF.batch_normalization(PF.convolution(h, 512, (3, 3), pad=(1, 1))) ) h = F.average_pooling(h, (2, 2)) with nn.parameter_scope("fc"): h = F.tanh(PF.affine(h, 1024)) with nn.parameter_scope("classifier"): y = PF.affine(h, 10) return y
def conv_lstm_cell(input_tensor, cur_state, n_filt, kernel_size): """ conv lstm cell definition """ def split(inp): _, channels, _, _ = inp.shape channels = channels / 4 return inp[:, :channels, :, :], inp[:, channels:2 * channels, :, :], \ inp[:, 2 * channels:3 * channels, :, :], \ inp[:, 3 * channels:4 * channels, :, :] h_cur, c_cur = cur_state # concatenate along channel axis combined = F.concatenate(*[input_tensor, h_cur], axis=1) combined_conv = conv2d(combined, 4 * n_filt, kernel_size, 1, kernel_size // 2, name='conv_lstm_cell') cc_i, cc_f, cc_o, cc_g = split(combined_conv) act_i = F.sigmoid(cc_i) act_f = F.sigmoid(cc_f) act_o = F.sigmoid(cc_o) act_g = F.tanh(cc_g) c_next = F.add2(act_f * c_cur, act_i * act_g) h_next = act_o * F.tanh(c_next) return h_next, c_next
def LSTMCell(x, h2, h1): units = h1.shape[1] #first stack h2=hidden, h1= cell h2 = F.concatenate(h2, x, axis=1) h3 = PF.affine(h2, (units), name='Affine') h4 = PF.affine(h2, (units), name='InputGate') h5 = PF.affine(h2, (units), name='ForgetGate') h6 = PF.affine(h2, (units), name='OutputGate') h3 = F.tanh(h3) h4 = F.sigmoid(h4) h5 = F.sigmoid(h5) h6 = F.sigmoid(h6) h4 = F.mul2(h4, h3) h5 = F.mul2(h5, h1) h4 = F.add2(h4, h5, True) h7 = F.tanh(h4) h6 = F.mul2(h6, h7) return h6, h4 # hidden, cell
def Generator_late(f_8, f_16, f_32, f_64, scope_name="Generator", train=True, img_size=1024, ngf=64, big=False): with nn.parameter_scope(scope_name): # Get number of channels nfc_multi = { 4: 16, 8: 8, 16: 4, 32: 2, 64: 2, 128: 1, 256: 0.5, 512: 0.25, 1024: 0.125 } nfc = {} for k, v in nfc_multi.items(): nfc[k] = int(v * ngf) def sn_w(w): return PF.spectral_norm(w, dim=0) f_128 = Upsample(f_64, nfc[128], "up64->128", train) f_128 = SLE(f_128, f_8, "sle8->128") f_256 = Upsample(f_128, nfc[256], "up128->256") f_256 = SLE(f_256, f_16, "sle16->256") f_last = f_256 if img_size > 256: if big: f_512 = UpsampleComp(f_256, nfc[512], "up256->512") else: f_512 = Upsample(f_256, nfc[512], "up256->512") f_512 = SLE(f_512, f_32, "sle32->512") f_last = f_512 if img_size > 512: f_1024 = Upsample(f_512, nfc[1024], "up512->1024") f_last = f_1024 # Conv + Tanh -> image img = F.tanh( PF.convolution(f_last, 3, (3, 3), pad=(1, 1), apply_w=sn_w, with_bias=False, name="conv_last")) img_small = F.tanh( PF.convolution(f_128, 3, (1, 1), apply_w=sn_w, with_bias=False, name="conv_last_small")) return [img, img_small]
def test_generate_tmp_nnp(): nn.clear_parameters() batch_size = 16 x0 = nn.Variable([batch_size, 100]) x1 = nn.Variable([batch_size, 100]) h1_0 = PF.affine(x0, 100, name='affine1_0') h1_1 = PF.affine(x1, 100, name='affine1_0') h1 = F.tanh(h1_0 + h1_1) h2 = F.tanh(PF.affine(h1, 50, name='affine2')) y0 = PF.affine(h2, 10, name='affiney_0') y1 = PF.affine(h2, 10, name='affiney_1') contents = { 'networks': [{ 'name': 'net1', 'batch_size': batch_size, 'outputs': { 'y0': y0, 'y1': y1 }, 'names': { 'x0': x0, 'x1': x1 } }], 'executors': [{ 'name': 'runtime', 'network': 'net1', 'data': ['x0', 'x1'], 'output': ['y0', 'y1'] }] } nnabla.utils.save.save('tmp.nnp', contents)
def gated_conv(self, x, kernel_shape, h=None, mask_type='', gated=True, payload=None, return_payload=False, scope_name='gated_conv'): pad_dim_0 = (kernel_shape[0] - 1) / 2 pad_dim_1 = (kernel_shape[1] - 1) / 2 if mask_type == '': mask_type = self.mask_type_B with nn.parameter_scope(scope_name): if gated: out_f = PF.convolution(x, self.num_features, kernel_shape, pad=(pad_dim_0, pad_dim_1), apply_w=mask_type, name='conv_f') out_g = PF.convolution(x, self.num_features, kernel_shape, pad=(pad_dim_0, pad_dim_1), apply_w=mask_type, name='conv_g') if isinstance(payload, nn.Variable): out_f += payload[:, :self.num_features, :, :] out_g += payload[:, self.num_features:, :, :] if self.conditional: h_out_f = PF.affine(h, self.num_features, name='h_out_f') h_out_f = h_out_f.reshape( (h_out_f.shape[0], h_out_f.shape[1], 1, 1)) h_out_g = PF.affine(h, self.num_features, name='h_out_g') h_out_g = h_out_g.reshape( (h_out_g.shape[0], h_out_g.shape[1], 1, 1)) out = F.tanh(out_f + h_out_f) * F.sigmoid(out_g + h_out_g) else: out = F.tanh(out_f) * F.sigmoid(out_g) if return_payload: payload = PF.convolution(F.concatenate(out_f, out_g, axis=1), 2 * self.num_features, (1, 1), name='conv_1x1') payload = F.relu(payload) return out, payload else: out = PF.convolution(x, self.num_features, kernel_shape, stride=(1, 1), pad=(pad_dim_0, pad_dim_1), apply_w=mask_type) out = F.relu(out) return out
def mlp_module(x0, x1): h1_0 = PF.affine(x0, 100, name='affine1_0') h1_1 = PF.affine(x1, 100, name='affine1_0') h1 = F.tanh(h1_0 + h1_1) h2 = F.tanh(PF.affine(h1, 50, name='affine2')) y0 = PF.affine(h2, 10, name='affiney_0') y1 = PF.affine(h2, 10, name='affiney_1') return y0, y1
def policy_network(obs, action_size, name): with nn.parameter_scope(name): out = PF.affine(obs, 64, name='fc1') out = F.tanh(out) out = PF.affine(out, 64, name='fc2') out = F.tanh(out) out = PF.affine(out, action_size, name='fc3') return F.tanh(out)
def q_network(obs, action, name): with nn.parameter_scope(name): out = PF.affine(obs, 64, name='fc1') out = F.tanh(out) out = F.concatenate(out, action, axis=1) out = PF.affine(out, 64, name='fc2') out = F.tanh(out) out = PF.affine(out, 1, name='fc3') return out
def lstm_cell(x, c, h): batch_size, units = c.shape _hidden = PF.affine(F.concatenate(x, h, axis=1), 4*units) a = F.tanh (F.slice(_hidden, start=(0, units*0), stop=(batch_size, units*1))) input_gate = F.sigmoid(F.slice(_hidden, start=(0, units*1), stop=(batch_size, units*2))) forgate_gate = F.sigmoid(F.slice(_hidden, start=(0, units*2), stop=(batch_size, units*3))) output_gate = F.sigmoid(F.slice(_hidden, start=(0, units*3), stop=(batch_size, units*4))) cell = input_gate * a + forgate_gate * c hidden = output_gate * F.tanh(cell) return cell, hidden
def lstm_cell(x: nn.Variable, c: nn.Variable, h: nn.Variable) -> nn.Variable: batch_size, units = c.shape _hidden = PF.affine(F.concatenate(x, h, axis=1), 4*units) a = F.tanh (_hidden[:, units*0: units*1]) input_gate = F.sigmoid(_hidden[:, units*1: units*2]) forgate_gate = F.sigmoid(_hidden[:, units*2: units*3]) output_gate = F.sigmoid(_hidden[:, units*3: units*4]) cell = input_gate * a + forgate_gate * c hidden = output_gate * F.tanh(cell) return cell, hidden
def network(x, maxh=16, depth=8): with nn.parameter_scope("net"): # (1, 28, 28) --> (32, 16, 16) with nn.parameter_scope("convIn"): out = F.tanh(PF.convolution(x, maxh, (1, 1), with_bias=False)) for i in range(depth): with nn.parameter_scope("conv" + str(i)): out = F.tanh(PF.convolution(out, maxh, (1, 1), with_bias=False)) with nn.parameter_scope("convOut"): out = F.sigmoid(PF.convolution(out, 3, (1, 1), with_bias=False)) return out
def LSTM(inputs, units, return_sequences=False, name='lstm'): ''' A long short-term memory layer Args: inputs (nnabla.Variable): A shape of [B, SentenceLength, EmbeddingSize]. units (int): Dimensionality of the output space. return_sequences (bool): Whether to return the last output. in the output sequence, or the full sequence. Returns: nn.Variable: A shape [B, SentenceLength, units]. or nn.Variable: A shape [B, units] ''' hs = [] batch_size = inputs.shape[0] sentence_length = inputs.shape[1] c0 = nn.Variable.from_numpy_array(np.zeros((batch_size, units))) h0 = nn.Variable.from_numpy_array(np.zeros((batch_size, units))) inputs = F.split(inputs, axis=1) cell = c0 hidden = h0 with nn.parameter_scope(name): for x in inputs: a = F.tanh( PF.affine(x, units, with_bias=False, name='Wa') + PF.affine(hidden, units, name='Ra')) input_gate = F.sigmoid( PF.affine(x, units, with_bias=False, name='Wi') + PF.affine(hidden, units, name='Ri')) forgate_gate = F.sigmoid( PF.affine(x, units, with_bias=False, name='Wf') + PF.affine(hidden, units, name='Rf')) cell = input_gate * a + forgate_gate * cell output_gate = F.sigmoid( PF.affine(x, units, with_bias=False, name='Wo') + PF.affine(hidden, units, name='Ro')) hidden = output_gate * F.tanh(cell) if return_sequences: hidden = F.reshape(hidden, (batch_size, 1, units)) hs.append(hidden) if return_sequences: hs = F.concatenate(*hs, axis=1) hs = F.reshape(hs, (batch_size, sentence_length, units)) return hs else: return hs[-1]
def network(x, y_index, test=False): # Input -> 3,64,64 # Convolution -> 16,31,31 with nn.parameter_scope('Convolution'): h = PF.convolution(x, 16, (3, 3), (0, 0), (2, 2)) # Tanh h = F.tanh(h) # MaxPooling -> 16,16,11 h = F.max_pooling(h, (2, 3), (2, 3)) # Dropout if not test: h = F.dropout(h) # Convolution_2 -> 32,6,5 with nn.parameter_scope('Convolution_2'): h = PF.convolution(h, 32, (5, 3), (0, 0), (2, 2)) # ReLU_4 h = F.relu(h, True) # MaxPooling_2 -> 32,3,3 h = F.max_pooling(h, (2, 2), (2, 2)) # Dropout_2 if not test: h = F.dropout(h) # Convolution_3 -> 64,1,1 with nn.parameter_scope('Convolution_3'): h = PF.convolution(h, 64, (3, 3), (0, 0), (2, 2)) # Tanh_2 h = F.tanh(h) # Dropout_3 if not test: h = F.dropout(h) # Affine -> 50 with nn.parameter_scope('Affine'): h = PF.affine(h, (50, )) # ReLU_2 h = F.relu(h, True) # Dropout_4 if not test: h = F.dropout(h) # Affine_2 -> 5 with nn.parameter_scope('Affine_2'): h = PF.affine(h, (5, )) # ELU h = F.elu(h) # Affine_3 -> 1 with nn.parameter_scope('Affine_3'): h = PF.affine(h, (1, )) # SquaredError #h = F.squared_error(h, y_index) return h
def lstm(x, h, c, w, b, with_bias): hidden_size = h.shape[1] xh = F.concatenate(*(x, h), axis=1) w0, w1, w2, w3 = F.split(w, axis=0) b0 = b1 = b2 = b3 = None if with_bias: b0, b1, b2, b3 = F.split(b, axis=0) i_t = F.affine(xh, F.transpose(w0, (1, 0)), b0) f_t = F.affine(xh, F.transpose(w1, (1, 0)), b1) g_t = F.affine(xh, F.transpose(w2, (1, 0)), b2) o_t = F.affine(xh, F.transpose(w3, (1, 0)), b3) c_t = F.sigmoid(f_t) * c + F.sigmoid(i_t) * F.tanh(g_t) h_t = F.sigmoid(o_t) * F.tanh(c_t) return h_t, c_t
def simple_rnn(inputs, units, return_sequences=False, fix_parameters=False): ''' A vanilla recurrent neural network layer Args: inputs (nnabla.Variable): A shape of [B, SentenceLength, EmbeddingSize]. units (int): Dimensionality of the output space. return_sequences (bool): Whether to return the last output. in the output sequence, or the full sequence. fix_parameters (bool): Fix parameters (Set need_grad=False). Returns: nn.Variable: A shape [B, SentenceLength, units]. or nn.Variable: A shape [B, units] ''' hs = [] batch_size = inputs.shape[0] sentence_length = inputs.shape[1] h0 = nn.Variable.from_numpy_array(np.zeros((batch_size, units))) inputs = F.split(inputs, axis=1) # split in the direction of sequence h = h0 for x in inputs: h = F.tanh(PF.affine(F.concatenate(x, h, axis=1), units, fix_parameters=fix_parameters)) hs.append(h) if return_sequences: hs = F.stack(*hs, axis=1) return hs else: return hs[-1]
def network(x): with nn.parameter_scope("cnn"): with nn.parameter_scope("conv1"): h = F.tanh( PF.batch_normalization( PF.convolution(x, 4, (3, 3), pad=(1, 1), stride=(2, 2)))) with nn.parameter_scope("conv2"): h = F.tanh( PF.batch_normalization(PF.convolution(h, 8, (3, 3), pad=(1, 1)))) h = F.average_pooling(h, (2, 2)) with nn.parameter_scope("fc3"): h = F.tanh(PF.affine(h, 16)) with nn.parameter_scope("classifier"): h = PF.affine(h, 5) return h
def Bahdanau_attention(query, values, out_features, scope): r"""Return the Bahdanau attention mechanism. Args: query (nn.Variable): A query of size (B, 1, C). values (nn.Variable): Values of size (B, T, C). out_features (int): The projected dimensionality. scope (str): Parameter scope. Returns: nn.Variable: The context vector. nn.Variable: The attention weight vector. """ with nn.parameter_scope(scope): x = PF.affine(query, out_features, base_axis=2, with_bias=False, name='query') y = PF.affine(values, out_features, base_axis=2, with_bias=False, name='values') # scores of shape (B, T, 1) scores = PF.affine(F.tanh(x + y), 1, base_axis=2, with_bias=False, name='scores') # attention_weights of shape (B, 1, T) attention_weights = F.softmax( scores, axis=1).reshape((query.shape[0], 1, -1)) # context_vector shape after sum == (B, 1, C) context_vector = F.batch_matmul(attention_weights, values) return context_vector, attention_weights
def SimpleDecoder(fea, scope_name): # Get number of channels nfc_multi = { 4: 16, 8: 8, 16: 4, 32: 2, 64: 2, 128: 1, 256: 0.5, 512: 0.25, 1024: 0.125 } nfc = {} for k, v in nfc_multi.items(): nfc[k] = int(v * 32) with nn.parameter_scope(scope_name): def sn_w(w): return PF.spectral_norm(w, dim=0) h = Upsample(fea, nfc[16], "up8->16") h = Upsample(h, nfc[32], "up16->32") h = Upsample(h, nfc[64], "up32->64") h = Upsample(h, nfc[128], "up64->128") h = PF.convolution(h, 3, (3, 3), pad=(1, 1), apply_w=sn_w, with_bias=False, name="conv1") img = F.tanh(h) return img
def _gru(x, h, w, b, with_bias): """GRU cell. Args: x (:obj:`~nnabla.Variable`): Input data. h (:obj:`~nnabla.Variable`): Hidden state. w (:obj:`~nnabla.Variable`): Weight. b (:obj:`~nnabla.Variable`): Bias. with_bias (bool): Include the bias or not. """ hidden_size = h.shape[1] xh = F.concatenate(*(x, h), axis=1) w0, w1, w2 = F.split(w, axis=0) b0 = b1 = b2 = b3 = None if with_bias: b0, b1, b2, b3 = F.split(b, axis=0) r_t = F.sigmoid(F.affine(xh, F.transpose(w0, (1, 0)), b0)) z_t = F.sigmoid(F.affine(xh, F.transpose(w1, (1, 0)), b1)) w2_0 = w2[:, :w2.shape[1] - hidden_size] w2_1 = w2[:, w2.shape[1] - hidden_size:] n_t = F.tanh( F.affine(x, F.transpose(w2_0, (1, 0)), b2) + r_t * F.affine(h, F.transpose(w2_1, (1, 0)), b3)) h_t = (1 - z_t) * n_t + z_t * h return h_t
def simple_rnn(inputs: nn.Variable, units: int, mask: Optional[nn.Variable] = None, return_sequences: bool = False, fix_parameters=False) -> nn.Variable: ''' A vanilla recurrent neural network layer Args: inputs (nnabla.Variable): A shape of [batch_size, length, embedding_size]. units (int): Dimensionality of the output space. mask (nnabla.Variable): A shape of [batch_size, length, 1]. return_sequences (bool): Whether to return the last output. in the output sequence, or the full sequence. fix_parameters (bool): Fix parameters (Set need_grad=False). Returns: nn.Variable: A shape [batch_size, length, units] or nn.Variable: A shape [batch_size units]. ''' hs = [] batch_size, length, embedding_size = inputs.shape h0 = F.constant(0, shape=(batch_size, units)) h = h0 if mask is None: mask = F.constant(1, shape=(batch_size, length, 1)) for x, cond in zip(F.split(inputs, axis=1), F.split(mask, axis=1)): h_t = F.tanh(PF.affine(F.concatenate(x, h, axis=1), units, fix_parameters=fix_parameters)) h = where(cond, h_t, h) hs.append(h) if return_sequences: hs = F.stack(*hs, axis=1) return hs else: return hs[-1]
def cnn(x): with nn.parameter_scope("cnn"): # Parameter scope can be nested with nn.parameter_scope("conv1"): c1 = F.tanh( PF.batch_normalization( PF.convolution(x, 4, (3, 3), pad=(1, 1), stride=(2, 2)))) with nn.parameter_scope("conv2"): c2 = F.tanh( PF.batch_normalization( PF.convolution(c1, 8, (3, 3), pad=(1, 1)))) c2 = F.average_pooling(c2, (2, 2)) with nn.parameter_scope("fc3"): fc3 = F.tanh(PF.affine(c2, 32)) with nn.parameter_scope("classifier"): y = PF.affine(fc3, 10) return y, [c1, c2, fc3]
def compute_context(prev_state): batch_size = prev_state.shape[0] ht = PF.affine(prev_state, attention_units, with_bias=False, name='Waht') # -> (batch_size, attention_units) ht = F.reshape(ht, (batch_size, 1, attention_units)) # -> (batch_size, 1, attention_units) ht = F.broadcast(ht, (batch_size, sentence_length_source, attention_units)) # -> (batch_size, sentence_length_source, attention_units) attention = F.tanh(hs + ht) # -> (batch_size, sentence_length_source, attention_units) attention = time_distributed(PF.affine)(attention, 1, with_bias=False, name='attention') # -> (batch_size, sentence_length_source, 1) attention = F.softmax(attention, axis=1) # -> (batch_size, sentence_length_source, 1) context = F.batch_matmul(hs, attention, transpose_a=True) context = F.reshape(context, (batch_size, attention_units)) return context
def decoder(content, style, maps=256, num_res=4, num_layers=2, pad_mode="reflect", name="decoder"): h = content styles = mlp(style, maps, num_res, num_layers) b, c, _, _ = h.shape with nn.parameter_scope("generator"): with nn.parameter_scope(name): for i in range(num_res): s = styles[:, i * maps * num_layers * 2:(i + 1) * maps * num_layers * 2] h = resblock(h, s, maps, norm="adain", pad_mode=pad_mode, name="resblock-{}".format(i + 1)) h = upsample(h, maps // 2, norm="ln", pad_mode=pad_mode, name="upsample-1") h = upsample(h, maps // 4, norm="ln", pad_mode=pad_mode, name="upsample-2") h = convolution(h, 3, 7, 3, 1, pad_mode=pad_mode, name="to-RGB") h = F.tanh(h) return h
def decode(input_feature, output_nc, n_downsampling, ngf, norm_layer, use_bias): h = input_feature w_init = I.NormalInitializer(sigma=0.02, rng=None) for i in range(n_downsampling): with nn.parameter_scope("dec_downsampling_{}".format(i)): mult = 2**(n_downsampling - i) h = PF.deconvolution(h, int(ngf * mult / 2), kernel=(4, 4), stride=(2, 2), pad=(1, 1), w_init=w_init, with_bias=use_bias) # kernel changed 3 -> 4 to make the output fit to the desired size. h = norm_layer(h) h = F.relu(h) h = F.pad(h, (3, 3, 3, 3), 'reflect') h = PF.convolution(h, output_nc, kernel=(7, 7), w_init=w_init, with_bias=use_bias, name="dec_last_conv") h = F.tanh(h) return h
def lighting_network(x_hat, normal, feature, view, D=512, L=4, N=4, including_input=True): """ Args x_hat: Differentiable intersection point (B, R, 3) normal: Normal on x_hat (B, R, 3) (should be normalized before). feature: Intermediate output of the implicit network (B, R, feature_size). view: View direction (B, R, 3) D: Dimension of a network. L: Number of layers. N: Number of frequency of the positional encoding. inclugin_input: Include input to the positional encoding (PE). """ pe_view = positional_encoding(view, N, including_input) h = F.concatenate(*[x_hat, normal, feature, pe_view], axis=-1) for l in range(L - 1): h = affine(h, D, name=f"affine-{l:02d}") h = F.relu(h) h = affine(h, 3, name=f"affine-{L - 1:02d}") h = F.tanh(h) return h
def generator(z, y, scopename="generator", maps=1024, n_classes=1000, s=4, test=False, sn=True, coefs=[1.0]): with nn.parameter_scope(scopename): # Affine h = affine(z, maps * s * s, with_bias=True, sn=sn, test=test) h = F.reshape(h, [h.shape[0]] + [maps, s, s]) # Resblocks h = resblock_g(h, y, "block-1", n_classes, maps // 1, test=test, sn=sn, coefs=coefs) h = resblock_g(h, y, "block-2", n_classes, maps // 2, test=test, sn=sn, coefs=coefs) h = resblock_g(h, y, "block-3", n_classes, maps // 4, test=test, sn=sn, coefs=coefs) h = attnblock(h, sn=sn, test=test) h = resblock_g(h, y, "block-4", n_classes, maps // 8, test=test, sn=sn, coefs=coefs) h = resblock_g(h, y, "block-5", n_classes, maps // 16, test=test, sn=sn, coefs=coefs) # Last convoltion h = BN(h, test=test) h = F.relu(h) h = convolution(h, 3, kernel=(3, 3), pad=(1, 1), stride=(1, 1), sn=sn, test=test) x = F.tanh(h) return x
def __call__(self, x, test=False): with nn.parameter_scope(self.scope): with nn.parameter_scope('conv1'): h1 = F.elu(bn(downsample(x, self.hidden_channel), test)) with nn.parameter_scope('conv2'): h2 = F.elu(bn(downsample(h1, self.hidden_channel // 8), test)) with nn.parameter_scope('conv3'): h3 = F.elu(bn(downsample(h2, self.hidden_channel // 4), test)) with nn.parameter_scope('conv4'): h4 = F.elu(bn(downsample(h3, self.hidden_channel // 2), test)) with nn.parameter_scope('deconv1'): h5 = F.elu(bn(upsample(h4, self.hidden_channel), test)) with nn.parameter_scope('deconv2'): h6 = F.elu(bn(upsample(h5, self.hidden_channel // 2), test)) with nn.parameter_scope('deconv3'): h7 = F.elu(bn(upsample(h6, self.hidden_channel // 4), test)) with nn.parameter_scope('deconv4'): h8 = F.elu(bn(upsample(h7, self.hidden_channel // 8), test)) with nn.parameter_scope('conv5'): y = F.tanh( PF.convolution(h8, self.out_channel, kernel=(3, 3), pad=(1, 1))) return y
def generator(x, scopename, maps=64, unpool=False, init_method=None): with nn.parameter_scope('generator'): with nn.parameter_scope(scopename): with nn.parameter_scope('conv1'): x = convblock(x, n=maps, k=(7, 7), s=(1, 1), p=(3, 3), leaky=False, init_method=init_method) with nn.parameter_scope('conv2'): x = convblock(x, n=maps*2, k=(3, 3), s=(2, 2), p=(1, 1), leaky=False, init_method=init_method) with nn.parameter_scope('conv3'): x = convblock(x, n=maps*4, k=(3, 3), s=(2, 2), p=(1, 1), leaky=False, init_method=init_method) for i in range(9): with nn.parameter_scope('res{}'.format(i+1)): x = resblock(x, n=maps*4, init_method=init_method) with nn.parameter_scope('deconv1'): x = unpool_block(x, n=maps*2, k=(4, 4), s=(2, 2), p=(1, 1), leaky=False, unpool=unpool, init_method=init_method) with nn.parameter_scope('deconv2'): x = unpool_block(x, n=maps, k=(4, 4), s=(2, 2), p=(1, 1), leaky=False, unpool=unpool, init_method=init_method) with nn.parameter_scope('conv4'): x = convolution(x, 3, kernel=(7, 7), stride=(1, 1), pad=(3, 3), init_method=init_method) x = F.tanh(x) return x
def test_sink(seed): rng = np.random.RandomState(seed) v = nn.Variable((2, 3, 4), need_grad=True) h0 = F.tanh(v) h1 = F.sigmoid(v) v.d = rng.randn(*v.shape).astype(np.float32) # Create references v.grad.zero() h0.forward() h1.forward() h0.backward() h1.backward() # v.grad is accumulated. h0d = h0.d.copy() h1d = h1.d.copy() vg = v.g.copy() # Reset values h0.data.zero() h1.data.zero() v.grad.zero() # Check if sink works dummy = F.sink(h0, h1, one_input_grad=True) dummy.forward() dummy.backward() assert np.all(h0d == h0.d) assert np.all(h1d == h1.d) assert np.all(vg == v.g)
def generator(z, maxh=256, test=False, output_hidden=False): """ Building generator network which takes (B, Z, 1, 1) inputs and generates (B, 1, 28, 28) outputs. """ # Define shortcut functions def bn(x): # Batch normalization return PF.batch_normalization(x, batch_stat=not test) def upsample2(x, c): # Twise upsampling with deconvolution. return PF.deconvolution(x, c, kernel=(4, 4), pad=(1, 1), stride=(2, 2), with_bias=False) assert maxh / 4 > 0 with nn.parameter_scope("gen"): # (Z, 1, 1) --> (256, 4, 4) with nn.parameter_scope("deconv1"): d1 = F.elu(bn(PF.deconvolution(z, maxh, (4, 4), with_bias=False))) # (256, 4, 4) --> (128, 8, 8) with nn.parameter_scope("deconv2"): d2 = F.elu(bn(upsample2(d1, maxh / 2))) # (128, 8, 8) --> (64, 16, 16) with nn.parameter_scope("deconv3"): d3 = F.elu(bn(upsample2(d2, maxh / 4))) # (64, 16, 16) --> (32, 28, 28) with nn.parameter_scope("deconv4"): # Convolution with kernel=4, pad=3 and stride=2 transforms a 28 x 28 map # to a 16 x 16 map. Deconvolution with those parameters behaves like an # inverse operation, i.e. maps 16 x 16 to 28 x 28. d4 = F.elu(bn(PF.deconvolution( d3, maxh / 8, (4, 4), pad=(3, 3), stride=(2, 2), with_bias=False))) # (32, 28, 28) --> (1, 28, 28) with nn.parameter_scope("conv5"): x = F.tanh(PF.convolution(d4, 1, (3, 3), pad=(1, 1))) if output_hidden: return x, [d1, d2, d3, d4] return x