def static_graph_case_1(self):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
if self.channel_last:
x = fluid.data("input", (-1, -1, -1, self.in_channels),
dtype=self.dtype)
else:
x = fluid.data("input", (-1, self.in_channels, -1, -1),
dtype=self.dtype)
y = fluid.layers.conv2d_transpose(
x,
self.out_channels,
output_size=self.output_size,
filter_size=self.filter_shape,
stride=self.stride,
padding=self.padding,
dilation=self.dilation,
groups=self.groups,
param_attr=I.NumpyArrayInitializer(self.weight),
bias_attr=False
if self.no_bias else I.NumpyArrayInitializer(self.bias),
data_format=self.data_format)
exe = fluid.Executor(self.place)
exe.run(start)
out, = exe.run(main, feed={"input": self.input}, fetch_list=[y])
return out
def fluid_layer(self, place):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
input_shape = (-1, -1, -1, -1, self.num_channels) \
if self.channel_last else (-1, self.num_channels, -1, -1, -1)
x_var = fluid.data("input", input_shape, dtype=self.dtype)
weight_attr = I.NumpyArrayInitializer(self.weight)
if self.bias is None:
bias_attr = False
else:
bias_attr = I.NumpyArrayInitializer(self.bias)
y_var = fluid.layers.conv3d_transpose(
x_var,
self.num_filters,
filter_size=self.filter_size,
output_size=self.output_size,
padding=self.padding,
stride=self.stride,
dilation=self.dilation,
groups=self.groups,
param_attr=weight_attr,
bias_attr=bias_attr,
use_cudnn=self.use_cudnn,
act=self.act,
data_format=self.data_format)
feed_dict = {"input": self.input}
exe = fluid.Executor(place)
exe.run(start)
y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var])
return y_np
def fluid_layer(self, place):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
x = fluid.data("input", [-1, self.feature_size],
dtype=self.dtype)
label = fluid.data("labels", [-1, 1], dtype="int64")
if self.is_custom:
path_table = fluid.data("path_table", [-1, -1],
dtype="int64")
path_code = fluid.data("path_code", [-1, -1],
dtype="int64")
else:
path_table = path_code = None
y = fluid.layers.hsigmoid(
x,
label,
self.num_classes,
param_attr=I.NumpyArrayInitializer(self.weight),
bias_attr=I.NumpyArrayInitializer(self.bias),
path_table=path_table,
path_code=path_code,
is_custom=self.is_custom,
is_sparse=self.is_sparse,
)
exe = fluid.Executor(place)
exe.run(start)
feed_dict = {"input": self.input, "labels": self.labels}
if self.is_custom:
feed_dict["path_code"] = self.path_code
feed_dict["path_table"] = self.path_table
y_np, = exe.run(main, feed=feed_dict, fetch_list=[y])
return y_np
def test_fluid_api(self):
train_program = fluid.Program()
startup_program = fluid.Program()
with fluid.program_guard(train_program, startup_program):
x = fluid.data('x', [-1, self.feature_size])
labels = fluid.data('labels', [-1, 1], 'int64')
path_table = None
path_code = None
if self.is_custom:
path_table = fluid.data('path_table', [-1, -1], 'int64')
path_code = fluid.data('path_code', [-1, -1], 'int64')
weight_attr = I.NumpyArrayInitializer(self.weight_np)
bias_attr = I.NumpyArrayInitializer(self.bias_np)
out = fluid.layers.hsigmoid(x, labels, self.num_classes,
weight_attr, bias_attr, 'out',
path_table, path_code, self.is_custom)
exe = fluid.Executor(self.place)
exe.run(startup_program)
feed_dict = {'x': self.x_np, 'labels': self.labels_np}
if self.is_custom:
feed_dict["path_code"] = self.path_code_np
feed_dict["path_table"] = self.path_table_np
ret, = exe.run(train_program, feed=feed_dict, fetch_list=[out])
self.assertTrue(np.allclose(ret, self.out_np))
def test_static_api(self):
train_program = paddle.static.Program()
startup_program = paddle.static.Program()
with paddle.static.program_guard(train_program, startup_program):
x = paddle.static.data('x', [-1, self.feature_size])
labels = paddle.static.data('labels', [-1, 1], 'int64')
weight = paddle.static.data('weight', [-1, self.feature_size])
bias = paddle.static.data('bias', [
-1,
])
path_table = None
path_code = None
if self.is_custom:
path_table = paddle.static.data('path_table', [-1, -1],
'int64')
path_code = paddle.static.data('path_code', [-1, -1], 'int64')
out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias,
path_table, path_code)
weight_attr = paddle.framework.ParamAttr(
initializer=I.NumpyArrayInitializer(self.weight_np))
bias_attr = paddle.framework.ParamAttr(
initializer=I.NumpyArrayInitializer(self.bias_np))
m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes,
weight_attr, bias_attr, self.is_custom)
out2 = m(x, labels, path_table, path_code)
exe = paddle.static.Executor(self.place)
exe.run(startup_program)
feed_dict = {
'x': self.x_np,
'labels': self.labels_np,
'weight': self.weight_np,
'bias': self.bias_np
}
if self.is_custom:
feed_dict["path_code"] = self.path_code_np
feed_dict["path_table"] = self.path_table_np
ret1, ret2 = exe.run(train_program,
feed=feed_dict,
fetch_list=[out1, out2])
for ret in [ret1, ret2]:
self.assertTrue(np.allclose(self.out_np, ret))
def nn_layer(self, place):
with dg.guard(place):
x_var = dg.to_variable(self.input)
conv = nn.RowConv(self.num_channels,
self.context_size,
param_attr=I.NumpyArrayInitializer(self.weight),
act=self.act,
dtype=self.dtype)
y_var = conv(x_var)
y_np = y_var.numpy()
return y_np
def nn_layer(self):
x_var = dg.to_variable(self.input)
label_var = dg.to_variable(self.labels)
if self.is_custom:
path_code_var = dg.to_variable(self.path_code)
path_table_var = dg.to_variable(self.path_table)
else:
path_code_var = path_table_var = None
hierarchical_softmax = nn.HSigmoid(
self.feature_size,
self.num_classes,
is_custom=self.is_custom,
param_attr=I.NumpyArrayInitializer(self.weight),
bias_attr=I.NumpyArrayInitializer(self.bias),
dtype=self.dtype)
y_var = hierarchical_softmax(x_var,
label_var,
path_table=path_table_var,
path_code=path_code_var)
y_np = y_var.numpy()
return y_np
def fluid_layer(self, place):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
input_shape = (-1, -1, -1,self.num_channels) \
if self.channel_last else (-1, self.num_channels, -1, -1)
x_var = fluid.data("input", input_shape, dtype=self.dtype)
weight_attr = I.NumpyArrayInitializer(self.weight)
if self.bias is None:
bias_attr = False
else:
bias_attr = I.NumpyArrayInitializer(self.bias)
if self.padding_mode != 'zeros':
x_var = F.pad(x_var,
self._reversed_padding_repeated_twice,
mode=self.padding_mode,
data_format=self.data_format)
padding = 0
else:
padding = self.padding
y_var = fluid.layers.conv2d(
x_var,
self.num_filters,
self.filter_size,
padding=padding,
stride=self.stride,
dilation=self.dilation,
groups=self.groups,
param_attr=weight_attr,
bias_attr=bias_attr,
data_format=self.data_format)
feed_dict = {"input": self.input}
exe = fluid.Executor(place)
exe.run(start)
y_np, = exe.run(main, feed=feed_dict, fetch_list=[y_var])
return y_np
def __init__(self,
attention_dim,
input_dim,
position_encoding_weight=1.,
position_rate=1.,
reduction_factor=1,
has_bias=False,
bias_dim=0,
keep_prob=1.):
super(AttentionBlock, self).__init__()
# positional encoding
omega_default = position_rate / reduction_factor
self.omega_default = omega_default
# multispeaker case
if has_bias:
std = np.sqrt(1.0 / bias_dim)
initializer = I.NormalInitializer(loc=0., scale=std)
self.q_pos_affine = dg.Linear(bias_dim, 1, param_attr=initializer)
self.k_pos_affine = dg.Linear(bias_dim, 1, param_attr=initializer)
self.omega_initial = self.create_parameter(
shape=[1], attr=I.ConstantInitializer(value=omega_default))
# mind the fact that q, k, v have the same feature dimension
# so we can init k_affine and q_affine's weight as the same matrix
# to get a better init attention
init_weight = np.random.normal(size=(input_dim, attention_dim),
scale=np.sqrt(1. / input_dim))
initializer = I.NumpyArrayInitializer(init_weight.astype(np.float32))
# 3 affine transformation to project q, k, v into attention_dim
q_affine = dg.Linear(input_dim, attention_dim, param_attr=initializer)
self.q_affine = weight_norm(q_affine, dim=-1)
k_affine = dg.Linear(input_dim, attention_dim, param_attr=initializer)
self.k_affine = weight_norm(k_affine, dim=-1)
std = np.sqrt(1.0 / input_dim)
initializer = I.NormalInitializer(loc=0., scale=std)
v_affine = dg.Linear(input_dim, attention_dim, param_attr=initializer)
self.v_affine = weight_norm(v_affine, dim=-1)
std = np.sqrt(1.0 / attention_dim)
initializer = I.NormalInitializer(loc=0., scale=std)
out_affine = dg.Linear(attention_dim,
input_dim,
param_attr=initializer)
self.out_affine = weight_norm(out_affine, dim=-1)
self.keep_prob = keep_prob
self.has_bias = has_bias
self.bias_dim = bias_dim
self.attention_dim = attention_dim
self.position_encoding_weight = position_encoding_weight
def static_graph_case(self):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
x = fluid.data("input", self.input.shape, dtype=paddle.float32)
y = fluid.layers.conv2d(
x,
self.num_filters,
self.filter_size,
stride=self.stride,
padding=self.padding,
dilation=self.dilation,
groups=self.groups,
param_attr=I.NumpyArrayInitializer(self.filter),
bias_attr=False if self.bias is None else
I.NumpyArrayInitializer(self.bias),
act=None,
data_format=self.data_format)
exe = fluid.Executor()
exe.run(start)
out, = exe.run(main, feed={"input": self.input}, fetch_list=[y])
return out
def test_dygraph_api(self):
paddle.disable_static(self.place)
x = paddle.to_tensor(self.x_np)
labels = paddle.to_tensor(self.labels_np)
weight = paddle.to_tensor(self.weight_np)
bias = paddle.to_tensor(self.bias_np)
path_table = None
path_code = None
if self.is_custom:
path_table = paddle.to_tensor(self.path_table_np)
path_code = paddle.to_tensor(self.path_code_np)
out1 = F.hsigmoid_loss(x, labels, self.num_classes, weight, bias,
path_table, path_code)
weight_attr = I.NumpyArrayInitializer(self.weight_np)
bias_attr = I.NumpyArrayInitializer(self.bias_np)
m = paddle.nn.HSigmoidLoss(self.feature_size, self.num_classes,
weight_attr, bias_attr, self.is_custom)
out2 = m(x, labels, path_table, path_code)
for out in [out1, out2]:
self.assertTrue(np.allclose(self.out_np, out.numpy()))
paddle.enable_static()
def fluid_layer(self, place):
main = fluid.Program()
start = fluid.Program()
with fluid.unique_name.guard():
with fluid.program_guard(main, start):
x = fluid.data("input", [-1, -1, self.num_channels],
dtype=self.dtype)
y = fluid.layers.row_conv(x,
self.context_size,
param_attr=I.NumpyArrayInitializer(
self.weight),
act=self.act)
exe = fluid.Executor(place)
exe.run(start)
y_np, = exe.run(main, feed={"input": self.input}, fetch_list=[y])
return y_np
def test_numpy_array_initializer(self):
"""Test the numpy array initializer with supplied arguments
"""
import numpy
program = framework.Program()
block = program.global_block()
np_array = numpy.random.random((10000)).astype("float32")
for _ in range(2):
block.create_parameter(
dtype=np_array.dtype,
shape=np_array.shape,
lod_level=0,
name="param",
initializer=initializer.NumpyArrayInitializer(np_array))
self.assertEqual(len(block.ops), 1)
init_op = block.ops[0]
self.assertEqual(init_op.type, 'assign_value')
assert (init_op.attr('fp32_values') == np_array).all()
def __init__(self, args, pretrained_embed=None):
super(Model, self).__init__()
self.args = args
# the embedding layer
self.word_embed = dygraph.Embedding(size=(args.n_words, args.n_embed))
if args.pretrained_embed_shape is not None:
if pretrained_embed is not None:
pre_param_attrs = fluid.ParamAttr(
name="pretrained_emb",
initializer=initializer.NumpyArrayInitializer(
pretrained_embed),
trainable=True)
self.pretrained = dygraph.Embedding(
size=args.pretrained_embed_shape,
param_attr=pre_param_attrs)
self.word_embed.weight = layers.create_parameter(
shape=(self.args.n_words, self.args.n_embed),
dtype='float32',
default_initializer=initializer.Constant(value=0.0))
else:
self.pretrained = dygraph.Embedding(
size=args.pretrained_embed_shape)
# Initialize feat feature, feat can be char or pos
if args.feat == 'char':
self.feat_embed = CharLSTM(n_chars=args.n_feats,
n_embed=args.n_char_embed,
n_out=args.n_feat_embed,
pad_index=args.feat_pad_index)
else:
self.feat_embed = dygraph.Embedding(size=(args.n_feats,
args.n_feat_embed))
self.embed_dropout = IndependentDropout(p=args.embed_dropout)
# lstm layer
self.lstm = BiLSTM(input_size=args.n_embed + args.n_feat_embed,
hidden_size=args.n_lstm_hidden,
num_layers=args.n_lstm_layers,
dropout=args.lstm_dropout)
self.lstm_dropout = SharedDropout(p=args.lstm_dropout)
# mlp layer
self.mlp_arc_h = MLP(n_in=args.n_lstm_hidden * 2,
n_out=args.n_mlp_arc,
dropout=args.mlp_dropout)
self.mlp_arc_d = MLP(n_in=args.n_lstm_hidden * 2,
n_out=args.n_mlp_arc,
dropout=args.mlp_dropout)
self.mlp_rel_h = MLP(n_in=args.n_lstm_hidden * 2,
n_out=args.n_mlp_rel,
dropout=args.mlp_dropout)
self.mlp_rel_d = MLP(n_in=args.n_lstm_hidden * 2,
n_out=args.n_mlp_rel,
dropout=args.mlp_dropout)
# biaffine layers
self.arc_attn = Biaffine(n_in=args.n_mlp_arc,
bias_x=True,
bias_y=False)
self.rel_attn = Biaffine(n_in=args.n_mlp_rel,
n_out=args.n_rels,
bias_x=True,
bias_y=True)
self.pad_index = args.pad_index
self.unk_index = args.unk_index
