def test_multiple_features(self):
batch_size = 2
data = []
for i in xrange(batch_size):
each_sample = []
each_sample.append(np.random.randint(10))
each_sample.append(
self.sparse_binary_reader(
20000, 40, non_empty=True))
each_sample.append(self.dense_reader(100))
data.append(each_sample)
# test multiple features
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0})
arg = feeder(data)
output_dense = arg.getSlotValue(0).copyToNumpyMat()
output_sparse = arg.getSlotValue(1)
output_index = arg.getSlotIds(2).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(output_dense[i].all(), data[i][2].all())
self.assertEqual(output_sparse.getSparseRowCols(i), data[i][1])
self.assertEqual(output_index[i], data[i][0])
# reader returns 3 features, but only use 2 features
data_types = [('fea0', data_type.dense_vector(100)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea2': 0})
arg = feeder(data)
output_dense = arg.getSlotValue(0).copyToNumpyMat()
output_index = arg.getSlotIds(1).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(output_dense[i].all(), data[i][2].all())
self.assertEqual(output_index[i], data[i][0])
# reader returns 3 featreus, one is duplicate data
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10)),
('fea3', data_type.dense_vector(100))]
feeder = DataFeeder(data_types,
{'fea0': 2,
'fea1': 1,
'fea2': 0,
'fea3': 2})
arg = feeder(data)
fea0 = arg.getSlotValue(0).copyToNumpyMat()
fea1 = arg.getSlotValue(1)
fea2 = arg.getSlotIds(2).copyToNumpyArray()
fea3 = arg.getSlotValue(3).copyToNumpyMat()
for i in xrange(batch_size):
self.assertEqual(fea0[i].all(), data[i][2].all())
self.assertEqual(fea1.getSparseRowCols(i), data[i][1])
self.assertEqual(fea2[i], data[i][0])
self.assertEqual(fea3[i].all(), data[i][2].all())
def test_multiple_features(self):
batch_size = 2
data = []
for i in xrange(batch_size):
each_sample = []
each_sample.append(np.random.randint(10))
each_sample.append(
self.sparse_binary_reader(20000, 40, non_empty=True))
each_sample.append(self.dense_reader(100))
data.append(each_sample)
# test multiple features
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0})
arg = feeder(data)
output_dense = arg.getSlotValue(0).copyToNumpyMat()
output_sparse = arg.getSlotValue(1)
output_index = arg.getSlotIds(2).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(output_dense[i].all(), data[i][2].all())
self.assertEqual(output_sparse.getSparseRowCols(i), data[i][1])
self.assertEqual(output_index[i], data[i][0])
# reader returns 3 features, but only use 2 features
data_types = [('fea0', data_type.dense_vector(100)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea2': 0})
arg = feeder(data)
output_dense = arg.getSlotValue(0).copyToNumpyMat()
output_index = arg.getSlotIds(1).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(output_dense[i].all(), data[i][2].all())
self.assertEqual(output_index[i], data[i][0])
# reader returns 3 featreus, one is duplicate data
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10)),
('fea3', data_type.dense_vector(100))]
feeder = DataFeeder(data_types, {
'fea0': 2,
'fea1': 1,
'fea2': 0,
'fea3': 2
})
arg = feeder(data)
fea0 = arg.getSlotValue(0).copyToNumpyMat()
fea1 = arg.getSlotValue(1)
fea2 = arg.getSlotIds(2).copyToNumpyArray()
fea3 = arg.getSlotValue(3).copyToNumpyMat()
for i in xrange(batch_size):
self.assertEqual(fea0[i].all(), data[i][2].all())
self.assertEqual(fea1.getSparseRowCols(i), data[i][1])
self.assertEqual(fea2[i], data[i][0])
self.assertEqual(fea3[i].all(), data[i][2].all())
def check_and_create_data(self):
"""
Checks if the input data is legal and creates the data layers
according to the input fields.
"""
if self.is_infer:
expected = ['q_ids', 'a_ids']
if len(self.inputs) < 2:
raise ValueError('''Input schema: expected vs given:
{} vs {}'''.format(expected, self.inputs))
else:
expected = ['q_ids', 'a_ids', 'label']
if len(self.inputs) < 3:
raise ValueError('''Input schema: expected vs given:
{} vs {}'''.format(expected, self.inputs))
self.label = layer.data(name=self.inputs[2],
type=data_type.integer_value(4))
self.q_ids = layer.data(
name=self.inputs[0],
type=data_type.integer_value_sequence(self.vocab_size))
self.a_ids = layer.data(
name=self.inputs[1],
type=data_type.integer_value_sequence(self.vocab_size))
def check_and_create_data(self):
"""
Checks if the input data is legal and creates the data layers
according to the input fields.
"""
if self.is_infer:
expected = ['q_ids', 'a_ids']
if len(self.inputs) < 2:
raise ValueError('''Input schema: expected vs given:
{} vs {}'''.format(expected, self.inputs))
else:
expected = ['q_ids', 'a_ids', 'label']
if len(self.inputs) < 3:
raise ValueError('''Input schema: expected vs given:
{} vs {}'''.format(expected, self.inputs))
self.label = layer.data(name=self.inputs[2],
type=data_type.integer_value(4))
self.q_ids = layer.data(name=self.inputs[0],
type=data_type.integer_value_sequence(
self.vocab_size))
self.a_ids = layer.data(name=self.inputs[1],
type=data_type.integer_value_sequence(
self.vocab_size))
def test_evaluator(self):
img = layer.data(name='pixel2', type=data_type.dense_vector(784))
output = layer.fc(input=img,
size=10,
act=activation.Softmax(),
name='fc_here')
lbl = layer.data(name='label2', type=data_type.integer_value(10))
cost = layer.cross_entropy_cost(input=output, label=lbl)
evaluator.classification_error(input=output, label=lbl)
print layer.parse_network(cost)
print layer.parse_network(output)
def test_evaluator(self):
img = layer.data(name='pixel2', type=data_type.dense_vector(784))
output = layer.fc(input=img,
size=10,
act=activation.Softmax(),
name='fc_here')
lbl = layer.data(name='label2', type=data_type.integer_value(10))
cost = layer.cross_entropy_cost(input=output, label=lbl)
evaluator.classification_error(input=output, label=lbl)
print layer.parse_network(cost)
print layer.parse_network(output)
def test_integer(self):
value_range = 100
batch_size = 32
index = []
for i in xrange(batch_size):
each_sample = []
each_sample.append(np.random.randint(value_range))
index.append(each_sample)
feeder = DataFeeder([('input', data_type.integer_value(value_range))],
{'input': 0})
arg = feeder(index)
output = arg.getSlotIds(0).copyToNumpyArray()
index = np.array(index, dtype='int')
self.assertEqual(output.all(), index.flatten().all())
def test_integer(self):
value_range = 100
batch_size = 32
index = []
for i in xrange(batch_size):
each_sample = []
each_sample.append(np.random.randint(value_range))
index.append(each_sample)
feeder = DataFeeder([('input', data_type.integer_value(value_range))],
{'input': 0})
arg = feeder(index)
output = arg.getSlotIds(0).copyToNumpyArray()
index = np.array(index, dtype='int')
self.assertEqual(output.all(), index.flatten().all())
def test_get_layer(self):
pixel = layer.data(name='pixel2', type=data_type.dense_vector(784))
label = layer.data(name='label2', type=data_type.integer_value(10))
hidden = layer.fc(input=pixel,
size=100,
act=conf_helps.SigmoidActivation())
inference = layer.fc(input=hidden,
size=10,
act=conf_helps.SoftmaxActivation())
cost = layer.classification_cost(input=inference, label=label)
topo = topology.Topology(cost)
pixel_layer = topo.get_layer("pixel2")
label_layer = topo.get_layer("label2")
self.assertEqual(pixel_layer, pixel)
self.assertEqual(label_layer, label)
def parse_new_rnn():
def new_step(y):
mem = layer.memory(name="rnn_state", size=hidden_dim)
out = layer.fc(input=[y, mem],
size=hidden_dim,
act=activation.Tanh(),
bias_attr=True,
name="rnn_state")
return out
data = layer.data(
name="word", type=data_type.integer_value(dict_dim))
embd = layer.embedding(input=data, size=word_dim)
rnn_layer = layer.recurrent_group(
name="rnn", step=new_step, input=embd)
return str(layer.parse_network(rnn_layer))
def parse_new_rnn():
def new_step(y):
mem = layer.memory(name="rnn_state", size=hidden_dim)
out = layer.fc(input=[y, mem],
size=hidden_dim,
act=activation.Tanh(),
bias_attr=True,
name="rnn_state")
return out
data = layer.data(name="word",
type=data_type.integer_value(dict_dim))
embd = layer.embedding(input=data, size=word_dim)
rnn_layer = layer.recurrent_group(name="rnn",
step=new_step,
input=embd)
return str(layer.parse_network(rnn_layer))
def test_parse(self):
pixel = layer.data(name='pixel3', type=data_type.dense_vector(784))
label = layer.data(name='label3', type=data_type.integer_value(10))
hidden = layer.fc(input=pixel,
size=100,
act=conf_helps.SigmoidActivation())
inference = layer.fc(input=hidden,
size=10,
act=conf_helps.SoftmaxActivation())
maxid = layer.max_id(input=inference)
cost1 = layer.classification_cost(input=inference, label=label)
cost2 = layer.cross_entropy_cost(input=inference, label=label)
topology.Topology(cost2).proto()
topology.Topology([cost1]).proto()
topology.Topology([cost1, cost2]).proto()
topology.Topology([inference, maxid]).proto()
def test_multiple_features_tuple(self):
batch_size = 2
data = []
for i in xrange(batch_size):
a = np.random.randint(10)
b = self.sparse_binary_reader(20000, 40, non_empty=True)
c = self.dense_reader(100)
each_sample = (a, b, c)
data.append(each_sample)
# test multiple features
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0})
arg = feeder(data)
out_dense = arg.getSlotValue(0).copyToNumpyMat()
out_sparse = arg.getSlotValue(1)
out_index = arg.getSlotIds(2).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(out_dense[i].all(), data[i][2].all())
self.assertEqual(out_sparse.getSparseRowCols(i), data[i][1])
self.assertEqual(out_index[i], data[i][0])
def test_multiple_features_tuple(self):
batch_size = 2
data = []
for i in xrange(batch_size):
a = np.random.randint(10)
b = self.sparse_binary_reader(20000, 40, non_empty=True)
c = self.dense_reader(100)
each_sample = (a, b, c)
data.append(each_sample)
# test multiple features
data_types = [('fea0', data_type.dense_vector(100)),
('fea1', data_type.sparse_binary_vector(20000)),
('fea2', data_type.integer_value(10))]
feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0})
arg = feeder(data)
out_dense = arg.getSlotValue(0).copyToNumpyMat()
out_sparse = arg.getSlotValue(1)
out_index = arg.getSlotIds(2).copyToNumpyArray()
for i in xrange(batch_size):
self.assertEqual(out_dense[i].all(), data[i][2].all())
self.assertEqual(out_sparse.getSparseRowCols(i), data[i][1])
self.assertEqual(out_index[i], data[i][0])
def test_data_type(self):
pixel = layer.data(name='pixel', type=data_type.dense_vector(784))
label = layer.data(name='label', type=data_type.integer_value(10))
hidden = layer.fc(input=pixel,
size=100,
act=conf_helps.SigmoidActivation())
inference = layer.fc(input=hidden,
size=10,
act=conf_helps.SoftmaxActivation())
cost = layer.classification_cost(input=inference, label=label)
topo = topology.Topology(cost)
data_types = topo.data_type()
self.assertEqual(len(data_types), 2)
pixel_data_type = filter(lambda type: type[0] == "pixel", data_types)
self.assertEqual(len(pixel_data_type), 1)
pixel_data_type = pixel_data_type[0]
self.assertEqual(pixel_data_type[1].type, pydp2.DataType.Dense)
self.assertEqual(pixel_data_type[1].dim, 784)
label_data_type = filter(lambda type: type[0] == "label", data_types)
self.assertEqual(len(label_data_type), 1)
label_data_type = label_data_type[0]
self.assertEqual(label_data_type[1].type, pydp2.DataType.Index)
self.assertEqual(label_data_type[1].dim, 10)
def test_recurrent_layer(self):
word = layer.data(name='word', type=data_type.integer_value(12))
recurrent = layer.recurrent(input=word)
lstm = layer.lstmemory(input=word)
gru = layer.grumemory(input=word)
print layer.parse_network(recurrent, lstm, gru)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle.v2.activation as activation
import paddle.v2.attr as attr
import paddle.v2.data_type as data_type
import paddle.v2.layer as layer
import paddle.v2.pooling as pooling
import paddle.v2.networks as networks
pixel = layer.data(name='pixel', type=data_type.dense_vector(128))
label = layer.data(name='label', type=data_type.integer_value(10))
weight = layer.data(name='weight', type=data_type.dense_vector(1))
combine_weight = layer.data(name='weight_combine',
type=data_type.dense_vector(10))
score = layer.data(name='score', type=data_type.dense_vector(1))
hidden = layer.fc(input=pixel,
size=100,
act=activation.Sigmoid(),
param_attr=attr.Param(name='hidden'))
inference = layer.fc(input=hidden, size=10, act=activation.Softmax())
conv = layer.img_conv(input=pixel,
filter_size=1,
filter_size_y=1,
num_channels=8,
num_filters=16,
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import paddle.v2.activation as activation
import paddle.v2.attr as attr
import paddle.v2.data_type as data_type
import paddle.v2.layer as layer
import paddle.v2.pooling as pooling
import paddle.v2.networks as networks
import paddle.v2.evaluator as evaluator
pixel = layer.data(name='pixel', type=data_type.dense_vector(128))
label = layer.data(name='label', type=data_type.integer_value(10))
weight = layer.data(name='weight', type=data_type.dense_vector(1))
combine_weight = layer.data(
name='weight_combine', type=data_type.dense_vector(10))
score = layer.data(name='score', type=data_type.dense_vector(1))
hidden = layer.fc(input=pixel,
size=100,
act=activation.Sigmoid(),
param_attr=attr.Param(name='hidden'))
inference = layer.fc(input=hidden, size=10, act=activation.Softmax())
conv = layer.img_conv(
input=pixel,
filter_size=1,
filter_size_y=1,
num_channels=8,
def test_recurrent_layer(self):
word = layer.data(name='word', type=data_type.integer_value(12))
recurrent = layer.recurrent(input=word)
lstm = layer.lstmemory(input=word)
gru = layer.grumemory(input=word)
print layer.parse_network([recurrent, lstm, gru])
