def StaticLenet(data, num_classes=10, classifier_activation='softmax'): conv2d_w1_attr = fluid.ParamAttr(name="conv2d_w_1") conv2d_w2_attr = fluid.ParamAttr(name="conv2d_w_2") fc_w1_attr = fluid.ParamAttr(name="fc_w_1") fc_w2_attr = fluid.ParamAttr(name="fc_w_2") fc_w3_attr = fluid.ParamAttr(name="fc_w_3") conv2d_b1_attr = fluid.ParamAttr(name="conv2d_b_1") conv2d_b2_attr = fluid.ParamAttr(name="conv2d_b_2") fc_b1_attr = fluid.ParamAttr(name="fc_b_1") fc_b2_attr = fluid.ParamAttr(name="fc_b_2") fc_b3_attr = fluid.ParamAttr(name="fc_b_3") conv1 = fluid.layers.conv2d(data, num_filters=6, filter_size=3, stride=1, padding=1, param_attr=conv2d_w1_attr, bias_attr=conv2d_b1_attr) batch_norm1 = layers.batch_norm(conv1) relu1 = layers.relu(batch_norm1) pool1 = fluid.layers.pool2d(relu1, pool_size=2, pool_type='max', pool_stride=2) conv2 = fluid.layers.conv2d(pool1, num_filters=16, filter_size=5, stride=1, padding=0, param_attr=conv2d_w2_attr, bias_attr=conv2d_b2_attr) batch_norm2 = layers.batch_norm(conv2) relu6_1 = layers.relu6(batch_norm2) pool2 = fluid.layers.pool2d(relu6_1, pool_size=2, pool_type='max', pool_stride=2) fc1 = fluid.layers.fc(input=pool2, size=120, param_attr=fc_w1_attr, bias_attr=fc_b1_attr) leaky_relu1 = layers.leaky_relu(fc1, alpha=0.01) fc2 = fluid.layers.fc(input=leaky_relu1, size=84, param_attr=fc_w2_attr, bias_attr=fc_b2_attr) sigmoid1 = layers.sigmoid(fc2) fc3 = fluid.layers.fc(input=sigmoid1, size=num_classes, param_attr=fc_w3_attr, bias_attr=fc_b3_attr) softmax1 = layers.softmax(fc3, use_cudnn=True) return softmax1
def mlp(self, features, name): h = features dim = features.shape[-1] dim_list = [dim * 2, dim] for i in range(2): h = L.fc(h, size=dim_list[i], name="%s_fc_%s" % (name, i), act=None) if self.args.norm_type == "layer_norm": log.info("norm_type is %s" % self.args.norm_type) h = L.layer_norm( h, begin_norm_axis=1, param_attr=F.ParamAttr( name="norm_scale_%s_%s" % (name, i), initializer=F.initializer.Constant(1.0)), bias_attr=F.ParamAttr( name="norm_bias_%s_%s" % (name, i), initializer=F.initializer.Constant(0.0)), ) else: log.info("using batch_norm") h = L.batch_norm(h) h = pgl.layers.graph_norm(self.graph_wrapper, h) h = L.relu(h) return h
def conv_bn_layer(self, input, num_filters, filter_size, stride, padding, name=None): """Create conv+bn layer""" conv = FL.conv2d(input=input, num_filters=num_filters, filter_size=filter_size, stride=stride, padding=padding, groups=1, act=None, param_attr=ParamAttr(name=name + "_weights"), bias_attr=False, name=name + '.conv2d.output.1') bn_name = name + ".bn" return FL.batch_norm( input=conv, act=None, name=bn_name + '.output.1', param_attr=ParamAttr(name=bn_name + '_scale'), bias_attr=ParamAttr(bn_name + '_offset'), moving_mean_name=bn_name + '_mean', moving_variance_name=bn_name + '_variance', )
def forward(self): """forward""" features_list = [self.gw.node_feat["attr"]] for i in range(self.num_layers): h = gin(self.gw, features_list[i], hidden_size=self.hidden_size, activation="relu", name="gin_%s" % (i), init_eps=0.0, train_eps=self.train_eps) h = fl.batch_norm(h) h = fl.relu(h) features_list.append(h) output = 0 for i, h in enumerate(features_list): pooled_h = pgl.layers.graph_pooling(self.gw, h, self.pool_type) drop_h = fl.dropout(pooled_h, self.dropout_prob, dropout_implementation="upscale_in_train") output += fl.fc(drop_h, size=self.num_class, act=None, param_attr=fluid.ParamAttr(name="final_fc_%s" % (i))) # calculate loss self.loss = fl.softmax_with_cross_entropy(output, self.labels) self.loss = fl.reduce_mean(self.loss) self.acc = fl.accuracy(fl.softmax(output), self.labels)
def conv2d_unit(x, filters, kernels, stride, padding, name, is_test, trainable): x = P.conv2d(input=x, num_filters=filters, filter_size=kernels, stride=stride, padding=padding, act=None, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name=name + ".conv.weights", trainable=trainable), bias_attr=False) bn_name = name + ".bn" x = P.batch_norm( input=x, act=None, is_test=is_test, param_attr=ParamAttr(initializer=fluid.initializer.Constant(1.0), regularizer=L2Decay(0.), trainable=trainable, name=bn_name + '.scale'), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), regularizer=L2Decay(0.), trainable=trainable, name=bn_name + '.offset'), moving_mean_name=bn_name + '.mean', moving_variance_name=bn_name + '.var') x = P.leaky_relu(x, alpha=0.1) return x
def forward(self, x): N, C, H, W = x.shape x_r = L.reshape(x, (1, N * C, H, W)) y_r = L.batch_norm(x_r, epsilon=self.eps, do_model_average_for_mean_and_var=False, use_global_stats=False) return L.reshape(y_r, (N, C, H, W))
def mlp(self, feat): for i in range(3): feat = L.fc(node, size=self.hidden_size, name="simple_mlp_{}".format(i)) feat = L.batch_norm(feat) feat = L.relu(feat) feat = L.dropout(feat, dropout_prob=0.5) return feat
def _DBL(input, num_filters, filter_size, padding=1, name=None): conv = pfl.conv2d(input=input, num_filters=num_filters, filter_size=filter_size, padding=padding, name=(name + '_conv2d') if name else None) bn = pfl.batch_norm(input=conv, name=(name + '_conv2d') if name else None) act = pfl.leaky_relu(bn, name=(name + '_act') if name else None) return act
def test_batch_norm(self): program = Program() with program_guard(program): data = layers.data(name='data', shape=[32, 128, 128], dtype="float32") out = layers.batch_norm(data) print(str(program))
def forward(self, x): if self.training: N, C, H, W = x.shape x_r = L.reshape(x, (1, N*C, H, W)) # for numeric stability y_r = L.batch_norm(x_r, epsilon=self.eps, do_model_average_for_mean_and_var=False, use_global_stats=False) return L.reshape(y_r, (N, C, H, W)) else: _mean = L.reduce_mean(x, dim=[2, 3], keep_dim=True) _var = var(x, dim=[2, 3], unbiased=False, keepdim=True) y = (x - _mean) / L.sqrt(_var + self.eps) return y
def forward(self, gw): x = self._atom_encoder(gw) patch_repr = [] for i in range(self.num_layers): e = self._bond_encoder(gw, name='l%d'%i) x = gin_layer(gw, x, e, 'gin_%s' % i) x = L.batch_norm( x, param_attr=F.ParamAttr(name='batchnorm_%s' % i)) patch_repr.append(x) # $h_i^{(k)}$ patch_summary = L.concat(patch_repr, axis=1) # $h_{\phi}^i$ patch_pool = [pgl.layers.graph_pooling(gw, x, 'sum') for x in patch_repr] global_repr = L.concat(patch_pool, axis=1) return global_repr, patch_summary
def __call__(self, input, is_test=False): return layers.batch_norm( input=input, act=act, is_test=is_test, momentum=momentum, epsilon=epsilon, param_attr=self.attr_holder.param_attr, bias_attr=self.attr_holder.bias_attr, data_layout=data_layout, in_place=in_place, name=name, moving_mean_name=self.attr_holder.moving_mean_attr.name, moving_variance_name=self.attr_holder.moving_variance_attr. name, do_model_average_for_mean_and_var= do_model_average_for_mean_and_var, use_global_stats=use_global_stats)
def conv_bn_layer(self, input, num_filters, filter_size, stride=1, groups=1, act=None, name=None): conv = layers.conv2d(input=input, num_filters=num_filters, filter_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, act=None, param_attr=ParamAttr(name=name + "_weights"), bias_attr=False) bn_name = "bn_" + name return layers.batch_norm(input=conv, act=act, param_attr=ParamAttr(name=bn_name + '_scale'), bias_attr=ParamAttr(bn_name + '_offset'), moving_mean_name=bn_name + '_mean', moving_variance_name=bn_name + '_variance')
def identity_block(input_tensor, filters, pre_name, is_test, trainable, use_dcn=False): filters1, filters2, filters3 = filters x = P.conv2d(input=input_tensor, num_filters=filters1, filter_size=1, stride=1, padding=0, act=None, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name=pre_name + ".conv1.weight", trainable=trainable), bias_attr=False) x = P.batch_norm( input=x, act=None, is_test=is_test, param_attr=ParamAttr(initializer=fluid.initializer.Constant(1.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn1.weight'), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn1.bias'), moving_mean_name=pre_name + '.bn1.running_mean', moving_variance_name=pre_name + '.bn1.running_var') x = P.relu(x) if use_dcn: offset_mask = P.conv2d( input=x, num_filters=27, filter_size=3, stride=1, padding=1, act=None, param_attr=ParamAttr( initializer=fluid.initializer.Normal(0.0, 0.01), name=pre_name + ".conv2.conv_offset_mask.weight", trainable=trainable), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), name=pre_name + ".conv2.conv_offset_mask.bias", trainable=trainable)) offset = offset_mask[:, :18, :, :] mask = offset_mask[:, 18:, :, :] mask = P.sigmoid(mask) x = P.deformable_conv( input=x, offset=offset, mask=mask, num_filters=filters2, filter_size=3, stride=1, padding=1, groups=1, deformable_groups=1, im2col_step=1, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name=pre_name + ".conv2.weight", trainable=trainable), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), name=pre_name + ".conv2.bias", trainable=trainable)) else: x = P.conv2d(input=x, num_filters=filters2, filter_size=3, stride=1, padding=1, act=None, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name=pre_name + ".conv2.weight", trainable=trainable), bias_attr=False) x = P.batch_norm( input=x, act=None, is_test=is_test, param_attr=ParamAttr(initializer=fluid.initializer.Constant(1.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn2.weight'), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn2.bias'), moving_mean_name=pre_name + '.bn2.running_mean', moving_variance_name=pre_name + '.bn2.running_var') x = P.relu(x) x = P.conv2d(input=x, num_filters=filters3, filter_size=1, stride=1, padding=0, act=None, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name=pre_name + ".conv3.weight", trainable=trainable), bias_attr=False) x = P.batch_norm( input=x, act=None, is_test=is_test, param_attr=ParamAttr(initializer=fluid.initializer.Constant(1.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn3.weight'), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), regularizer=L2Decay(0.), trainable=trainable, name=pre_name + '.bn3.bias'), moving_mean_name=pre_name + '.bn3.running_mean', moving_variance_name=pre_name + '.bn3.running_var') x = P.elementwise_add(x=x, y=input_tensor, act='relu') return x
def Resnet101(inputs, is_test, trainable, use_dcn): x = P.conv2d(inputs, 64, filter_size=7, stride=2, padding=3, param_attr=ParamAttr(initializer=fluid.initializer.Normal( 0.0, 0.01), name="backbone.conv1.weight", trainable=trainable), bias_attr=False) x = P.batch_norm( input=x, act=None, is_test=is_test, param_attr=ParamAttr(initializer=fluid.initializer.Constant(1.0), regularizer=L2Decay(0.), trainable=trainable, name='backbone.bn1.weight'), bias_attr=ParamAttr(initializer=fluid.initializer.Constant(0.0), regularizer=L2Decay(0.), trainable=trainable, name='backbone.bn1.bias'), moving_mean_name='backbone.bn1.running_mean', moving_variance_name='backbone.bn1.running_var') x = P.relu(x) x = P.pool2d(x, pool_size=3, pool_type="max", pool_stride=2, pool_padding=1) # stage2 x = conv_block(x, [64, 64, 256], 'backbone.layers.0.0', is_test, trainable, stride=1) x = identity_block(x, [64, 64, 256], 'backbone.layers.0.1', is_test, trainable) x = identity_block(x, [64, 64, 256], 'backbone.layers.0.2', is_test, trainable) # stage3 x = conv_block(x, [128, 128, 512], 'backbone.layers.1.0', is_test, trainable, use_dcn=use_dcn) x = identity_block(x, [128, 128, 512], 'backbone.layers.1.1', is_test, trainable, use_dcn=use_dcn) x = identity_block(x, [128, 128, 512], 'backbone.layers.1.2', is_test, trainable, use_dcn=use_dcn) s8 = identity_block(x, [128, 128, 512], 'backbone.layers.1.3', is_test, trainable, use_dcn=use_dcn) # stage4 x = conv_block(s8, [256, 256, 1024], 'backbone.layers.2.0', is_test, trainable, use_dcn=use_dcn) for i in range(1, 22): x = identity_block(x, [256, 256, 1024], 'backbone.layers.2.%d' % i, is_test, trainable, use_dcn=use_dcn) s16 = identity_block(x, [256, 256, 1024], 'backbone.layers.2.22', is_test, trainable, use_dcn=use_dcn) # stage5 x = conv_block(s16, [512, 512, 2048], 'backbone.layers.3.0', is_test, trainable, use_dcn=use_dcn) x = identity_block(x, [512, 512, 2048], 'backbone.layers.3.1', is_test, trainable, use_dcn=use_dcn) s32 = identity_block(x, [512, 512, 2048], 'backbone.layers.3.2', is_test, trainable, use_dcn=use_dcn) return s8, s16, s32
def forward(self, graph_wrapper, is_test=False): """ Build the network. """ node_features = self._mol_encoder(graph_wrapper, name=self.name) features_list = [node_features] for layer in range(self.layer_num): edge_features = self._bond_encoder( graph_wrapper, name='%s_layer%s' % (self.name, layer)) if self.gnn_type == "gcn": feat = gcn_layer( graph_wrapper, features_list[layer], edge_features, act="relu", name="%s_layer%s_gcn" % (self.name, layer)) elif self.gnn_type == "gat": feat = gat_layer( graph_wrapper, features_list[layer], edge_features, self.embed_dim, act="relu", name="%s_layer%s_gat" % (self.name, layer)) else: feat = gin_layer( graph_wrapper, features_list[layer], edge_features, name="%s_layer%s_gin" % (self.name, layer)) if self.norm_type == 'batch_norm': feat = layers.batch_norm( feat, param_attr=fluid.ParamAttr( name="%s_layer%s_batch_norm_scale" % (self.name, layer), initializer=fluid.initializer.Constant(1.0)), bias_attr=fluid.ParamAttr( name="%s_layer%s_batch_norm_bias" % (self.name, layer), initializer=fluid.initializer.Constant(0.0)), moving_mean_name="%s_layer%s_batch_norm_moving_avearage" % (self.name, layer), moving_variance_name="%s_layer%s_batch_norm_moving_variance" % (self.name, layer), is_test=is_test) elif self.norm_type == 'layer_norm': feat = layers.layer_norm( feat, param_attr=fluid.ParamAttr( name="%s_layer%s_layer_norm_scale" % (self.name, layer), initializer=fluid.initializer.Constant(1.0)), bias_attr=fluid.ParamAttr( name="%s_layer%s_layer_norm_bias" % (self.name, layer), initializer=fluid.initializer.Constant(0.0))) else: raise ValueError('%s not supported.' % self.norm_type) if self.graph_norm: feat = pgl.layers.graph_norm(graph_wrapper, feat) if layer < self.layer_num - 1: feat = layers.relu(feat) feat = layers.dropout( feat, self.dropout_rate, dropout_implementation="upscale_in_train", is_test=is_test) # residual if self.residual: feat = feat + features_list[layer] features_list.append(feat) if self.JK == "sum": node_repr = layers.reduce_sum(features_list, axis=0) elif self.JK == "mean": node_repr = layers.reduce_mean(features_list, axis=0) elif self.JK == "last": node_repr = features_list[-1] else: node_repr = features_list[-1] return node_repr
def up_sampling_2(x, num_filters, name, act='relu'): x = pixel_shuffle(x, 2) x = conv2d(x, num_filters, 3, padding=1, name=name + "_conv2d_1") x = batch_norm(x, act=act, name=name + "_bn") return x
def down_sampling_2(x, num_filters, name, act='leaky_relu'): x = conv2d(x, num_filters, 3, stride=2, padding=1, name=name + "_conv2d") x = batch_norm(x, act=act, name=name + "_bn") x = dropout(x, 0.25, name=name + "_dropout") return x