def test_multiple_features(self): batch_size = 2 data = [] for i in xrange(batch_size): each_sample = [] each_sample.append(np.random.randint(10)) each_sample.append( self.sparse_binary_reader( 20000, 40, non_empty=True)) each_sample.append(self.dense_reader(100)) data.append(each_sample) # test multiple features data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0}) arg = feeder(data) output_dense = arg.getSlotValue(0).copyToNumpyMat() output_sparse = arg.getSlotValue(1) output_index = arg.getSlotIds(2).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(output_dense[i].all(), data[i][2].all()) self.assertEqual(output_sparse.getSparseRowCols(i), data[i][1]) self.assertEqual(output_index[i], data[i][0]) # reader returns 3 features, but only use 2 features data_types = [('fea0', data_type.dense_vector(100)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea2': 0}) arg = feeder(data) output_dense = arg.getSlotValue(0).copyToNumpyMat() output_index = arg.getSlotIds(1).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(output_dense[i].all(), data[i][2].all()) self.assertEqual(output_index[i], data[i][0]) # reader returns 3 featreus, one is duplicate data data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10)), ('fea3', data_type.dense_vector(100))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0, 'fea3': 2}) arg = feeder(data) fea0 = arg.getSlotValue(0).copyToNumpyMat() fea1 = arg.getSlotValue(1) fea2 = arg.getSlotIds(2).copyToNumpyArray() fea3 = arg.getSlotValue(3).copyToNumpyMat() for i in xrange(batch_size): self.assertEqual(fea0[i].all(), data[i][2].all()) self.assertEqual(fea1.getSparseRowCols(i), data[i][1]) self.assertEqual(fea2[i], data[i][0]) self.assertEqual(fea3[i].all(), data[i][2].all())
def test_multiple_features(self): batch_size = 2 data = [] for i in xrange(batch_size): each_sample = [] each_sample.append(np.random.randint(10)) each_sample.append( self.sparse_binary_reader(20000, 40, non_empty=True)) each_sample.append(self.dense_reader(100)) data.append(each_sample) # test multiple features data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0}) arg = feeder(data) output_dense = arg.getSlotValue(0).copyToNumpyMat() output_sparse = arg.getSlotValue(1) output_index = arg.getSlotIds(2).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(output_dense[i].all(), data[i][2].all()) self.assertEqual(output_sparse.getSparseRowCols(i), data[i][1]) self.assertEqual(output_index[i], data[i][0]) # reader returns 3 features, but only use 2 features data_types = [('fea0', data_type.dense_vector(100)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea2': 0}) arg = feeder(data) output_dense = arg.getSlotValue(0).copyToNumpyMat() output_index = arg.getSlotIds(1).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(output_dense[i].all(), data[i][2].all()) self.assertEqual(output_index[i], data[i][0]) # reader returns 3 featreus, one is duplicate data data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10)), ('fea3', data_type.dense_vector(100))] feeder = DataFeeder(data_types, { 'fea0': 2, 'fea1': 1, 'fea2': 0, 'fea3': 2 }) arg = feeder(data) fea0 = arg.getSlotValue(0).copyToNumpyMat() fea1 = arg.getSlotValue(1) fea2 = arg.getSlotIds(2).copyToNumpyArray() fea3 = arg.getSlotValue(3).copyToNumpyMat() for i in xrange(batch_size): self.assertEqual(fea0[i].all(), data[i][2].all()) self.assertEqual(fea1.getSparseRowCols(i), data[i][1]) self.assertEqual(fea2[i], data[i][0]) self.assertEqual(fea3[i].all(), data[i][2].all())
def check_and_create_data(self): """ Checks if the input data is legal and creates the data layers according to the input fields. """ if self.is_infer: expected = ['q_ids', 'a_ids'] if len(self.inputs) < 2: raise ValueError('''Input schema: expected vs given: {} vs {}'''.format(expected, self.inputs)) else: expected = ['q_ids', 'a_ids', 'label'] if len(self.inputs) < 3: raise ValueError('''Input schema: expected vs given: {} vs {}'''.format(expected, self.inputs)) self.label = layer.data(name=self.inputs[2], type=data_type.integer_value(4)) self.q_ids = layer.data( name=self.inputs[0], type=data_type.integer_value_sequence(self.vocab_size)) self.a_ids = layer.data( name=self.inputs[1], type=data_type.integer_value_sequence(self.vocab_size))
def check_and_create_data(self): """ Checks if the input data is legal and creates the data layers according to the input fields. """ if self.is_infer: expected = ['q_ids', 'a_ids'] if len(self.inputs) < 2: raise ValueError('''Input schema: expected vs given: {} vs {}'''.format(expected, self.inputs)) else: expected = ['q_ids', 'a_ids', 'label'] if len(self.inputs) < 3: raise ValueError('''Input schema: expected vs given: {} vs {}'''.format(expected, self.inputs)) self.label = layer.data(name=self.inputs[2], type=data_type.integer_value(4)) self.q_ids = layer.data(name=self.inputs[0], type=data_type.integer_value_sequence( self.vocab_size)) self.a_ids = layer.data(name=self.inputs[1], type=data_type.integer_value_sequence( self.vocab_size))
def test_evaluator(self): img = layer.data(name='pixel2', type=data_type.dense_vector(784)) output = layer.fc(input=img, size=10, act=activation.Softmax(), name='fc_here') lbl = layer.data(name='label2', type=data_type.integer_value(10)) cost = layer.cross_entropy_cost(input=output, label=lbl) evaluator.classification_error(input=output, label=lbl) print layer.parse_network(cost) print layer.parse_network(output)
def test_evaluator(self): img = layer.data(name='pixel2', type=data_type.dense_vector(784)) output = layer.fc(input=img, size=10, act=activation.Softmax(), name='fc_here') lbl = layer.data(name='label2', type=data_type.integer_value(10)) cost = layer.cross_entropy_cost(input=output, label=lbl) evaluator.classification_error(input=output, label=lbl) print layer.parse_network(cost) print layer.parse_network(output)
def test_integer(self): value_range = 100 batch_size = 32 index = [] for i in xrange(batch_size): each_sample = [] each_sample.append(np.random.randint(value_range)) index.append(each_sample) feeder = DataFeeder([('input', data_type.integer_value(value_range))], {'input': 0}) arg = feeder(index) output = arg.getSlotIds(0).copyToNumpyArray() index = np.array(index, dtype='int') self.assertEqual(output.all(), index.flatten().all())
def test_integer(self): value_range = 100 batch_size = 32 index = [] for i in xrange(batch_size): each_sample = [] each_sample.append(np.random.randint(value_range)) index.append(each_sample) feeder = DataFeeder([('input', data_type.integer_value(value_range))], {'input': 0}) arg = feeder(index) output = arg.getSlotIds(0).copyToNumpyArray() index = np.array(index, dtype='int') self.assertEqual(output.all(), index.flatten().all())
def test_get_layer(self): pixel = layer.data(name='pixel2', type=data_type.dense_vector(784)) label = layer.data(name='label2', type=data_type.integer_value(10)) hidden = layer.fc(input=pixel, size=100, act=conf_helps.SigmoidActivation()) inference = layer.fc(input=hidden, size=10, act=conf_helps.SoftmaxActivation()) cost = layer.classification_cost(input=inference, label=label) topo = topology.Topology(cost) pixel_layer = topo.get_layer("pixel2") label_layer = topo.get_layer("label2") self.assertEqual(pixel_layer, pixel) self.assertEqual(label_layer, label)
def parse_new_rnn(): def new_step(y): mem = layer.memory(name="rnn_state", size=hidden_dim) out = layer.fc(input=[y, mem], size=hidden_dim, act=activation.Tanh(), bias_attr=True, name="rnn_state") return out data = layer.data( name="word", type=data_type.integer_value(dict_dim)) embd = layer.embedding(input=data, size=word_dim) rnn_layer = layer.recurrent_group( name="rnn", step=new_step, input=embd) return str(layer.parse_network(rnn_layer))
def parse_new_rnn(): def new_step(y): mem = layer.memory(name="rnn_state", size=hidden_dim) out = layer.fc(input=[y, mem], size=hidden_dim, act=activation.Tanh(), bias_attr=True, name="rnn_state") return out data = layer.data(name="word", type=data_type.integer_value(dict_dim)) embd = layer.embedding(input=data, size=word_dim) rnn_layer = layer.recurrent_group(name="rnn", step=new_step, input=embd) return str(layer.parse_network(rnn_layer))
def test_parse(self): pixel = layer.data(name='pixel3', type=data_type.dense_vector(784)) label = layer.data(name='label3', type=data_type.integer_value(10)) hidden = layer.fc(input=pixel, size=100, act=conf_helps.SigmoidActivation()) inference = layer.fc(input=hidden, size=10, act=conf_helps.SoftmaxActivation()) maxid = layer.max_id(input=inference) cost1 = layer.classification_cost(input=inference, label=label) cost2 = layer.cross_entropy_cost(input=inference, label=label) topology.Topology(cost2).proto() topology.Topology([cost1]).proto() topology.Topology([cost1, cost2]).proto() topology.Topology([inference, maxid]).proto()
def test_multiple_features_tuple(self): batch_size = 2 data = [] for i in xrange(batch_size): a = np.random.randint(10) b = self.sparse_binary_reader(20000, 40, non_empty=True) c = self.dense_reader(100) each_sample = (a, b, c) data.append(each_sample) # test multiple features data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0}) arg = feeder(data) out_dense = arg.getSlotValue(0).copyToNumpyMat() out_sparse = arg.getSlotValue(1) out_index = arg.getSlotIds(2).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(out_dense[i].all(), data[i][2].all()) self.assertEqual(out_sparse.getSparseRowCols(i), data[i][1]) self.assertEqual(out_index[i], data[i][0])
def test_multiple_features_tuple(self): batch_size = 2 data = [] for i in xrange(batch_size): a = np.random.randint(10) b = self.sparse_binary_reader(20000, 40, non_empty=True) c = self.dense_reader(100) each_sample = (a, b, c) data.append(each_sample) # test multiple features data_types = [('fea0', data_type.dense_vector(100)), ('fea1', data_type.sparse_binary_vector(20000)), ('fea2', data_type.integer_value(10))] feeder = DataFeeder(data_types, {'fea0': 2, 'fea1': 1, 'fea2': 0}) arg = feeder(data) out_dense = arg.getSlotValue(0).copyToNumpyMat() out_sparse = arg.getSlotValue(1) out_index = arg.getSlotIds(2).copyToNumpyArray() for i in xrange(batch_size): self.assertEqual(out_dense[i].all(), data[i][2].all()) self.assertEqual(out_sparse.getSparseRowCols(i), data[i][1]) self.assertEqual(out_index[i], data[i][0])
def test_data_type(self): pixel = layer.data(name='pixel', type=data_type.dense_vector(784)) label = layer.data(name='label', type=data_type.integer_value(10)) hidden = layer.fc(input=pixel, size=100, act=conf_helps.SigmoidActivation()) inference = layer.fc(input=hidden, size=10, act=conf_helps.SoftmaxActivation()) cost = layer.classification_cost(input=inference, label=label) topo = topology.Topology(cost) data_types = topo.data_type() self.assertEqual(len(data_types), 2) pixel_data_type = filter(lambda type: type[0] == "pixel", data_types) self.assertEqual(len(pixel_data_type), 1) pixel_data_type = pixel_data_type[0] self.assertEqual(pixel_data_type[1].type, pydp2.DataType.Dense) self.assertEqual(pixel_data_type[1].dim, 784) label_data_type = filter(lambda type: type[0] == "label", data_types) self.assertEqual(len(label_data_type), 1) label_data_type = label_data_type[0] self.assertEqual(label_data_type[1].type, pydp2.DataType.Index) self.assertEqual(label_data_type[1].dim, 10)
def test_recurrent_layer(self): word = layer.data(name='word', type=data_type.integer_value(12)) recurrent = layer.recurrent(input=word) lstm = layer.lstmemory(input=word) gru = layer.grumemory(input=word) print layer.parse_network(recurrent, lstm, gru)
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import paddle.v2.activation as activation import paddle.v2.attr as attr import paddle.v2.data_type as data_type import paddle.v2.layer as layer import paddle.v2.pooling as pooling import paddle.v2.networks as networks pixel = layer.data(name='pixel', type=data_type.dense_vector(128)) label = layer.data(name='label', type=data_type.integer_value(10)) weight = layer.data(name='weight', type=data_type.dense_vector(1)) combine_weight = layer.data(name='weight_combine', type=data_type.dense_vector(10)) score = layer.data(name='score', type=data_type.dense_vector(1)) hidden = layer.fc(input=pixel, size=100, act=activation.Sigmoid(), param_attr=attr.Param(name='hidden')) inference = layer.fc(input=hidden, size=10, act=activation.Softmax()) conv = layer.img_conv(input=pixel, filter_size=1, filter_size_y=1, num_channels=8, num_filters=16,
# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import unittest import paddle.v2.activation as activation import paddle.v2.attr as attr import paddle.v2.data_type as data_type import paddle.v2.layer as layer import paddle.v2.pooling as pooling import paddle.v2.networks as networks import paddle.v2.evaluator as evaluator pixel = layer.data(name='pixel', type=data_type.dense_vector(128)) label = layer.data(name='label', type=data_type.integer_value(10)) weight = layer.data(name='weight', type=data_type.dense_vector(1)) combine_weight = layer.data( name='weight_combine', type=data_type.dense_vector(10)) score = layer.data(name='score', type=data_type.dense_vector(1)) hidden = layer.fc(input=pixel, size=100, act=activation.Sigmoid(), param_attr=attr.Param(name='hidden')) inference = layer.fc(input=hidden, size=10, act=activation.Softmax()) conv = layer.img_conv( input=pixel, filter_size=1, filter_size_y=1, num_channels=8,
def test_recurrent_layer(self): word = layer.data(name='word', type=data_type.integer_value(12)) recurrent = layer.recurrent(input=word) lstm = layer.lstmemory(input=word) gru = layer.grumemory(input=word) print layer.parse_network([recurrent, lstm, gru])