Example #1
0
    def test_copy_samples_subset_with_data_func(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        assert_greater(len(x), 0, "This test needs non empty data.")
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset_with_data_func_lmdb')
        keys = range(0, len(x), 2)
        assert_greater(len(keys), 0, "This test needs a non-empty subset.")
        assert_greater(len(x), len(keys), "Need subset, not all elements.")

        ALPHA = -10  # non-zero
        def func_data_mul(value):
            _, x = r.unpack_raw_datum(value)
            dat = caffe.io.array_to_datum(ALPHA * x, int(x.flatten()[0]))
            return dat.SerializeToString()

        c.copy_samples_lmdb(path_src, path_dst, keys, func_data=func_data_mul)
        assert_true(os.path.isdir(path_dst), "failed to save LMDB")

        y = r.read_values(path_dst)
        assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
        for (x_val, x_label), (y_val, y_label) in zip(x[0::2], y):  # skip element in x
            assert_true(np.all(x_val * ALPHA == y_val), "Wrong content copied.")
            assert_true(np.all(int(x_val.flatten()[0]) == y_label), "Wrong content copied for label.")
            assert_false(np.all(x_label == y_label), "Wrong content copied for label.")
Example #2
0
    def test_copy_samples_all(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_all_lmdb')
        c.copy_samples_lmdb(path_src, path_dst, range(len(x)))
        assert_true(os.path.isdir(path_dst), "failed to save LMDB")

        y = r.read_values(path_dst)
        assert_equal(len(x), len(y), "Wrong number of elements copied.")
        for a, b in zip(x, y):
            assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
            assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
Example #3
0
    def test_read_values_float(self, mock_dat):

        img2_data = self.img1_data[2:, 1:, :]

        # mocks
        df = [
            self.img1_data.astype(float).flatten().ravel().tolist(),
            img2_data.astype(float).flatten().ravel().tolist()
        ]

        mock_dat.return_value.ParseFromString.return_value = ""
        type(mock_dat.return_value).data = PropertyMock(return_value='')
        type(mock_dat.return_value).float_data = PropertyMock(side_effect=df)
        type(mock_dat.return_value).channels = PropertyMock(side_effect=[4, 2])
        type(mock_dat.return_value).height = PropertyMock(side_effect=[2, 1])
        type(mock_dat.return_value).width = PropertyMock(side_effect=[3, 3])

        v = r.read_values(self.path_lmdb, float)

        assert_equal(len(v), 2, "Unexpected no. of elements.")

        x, _ = v[0]
        for ch in range(4):
            for row in range(2):
                for col in range(3):
                    assert_equal(x[ch, row, col], self.img1_data[ch, row, col])

        x, _ = v[1]

        for ch in range(2):
            for row in range(1):
                for col in range(3):
                    assert_equal(x[ch, row, col], img2_data[ch, row, col])
Example #4
0
    def test_copy_samples_single_reverse(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_single_lmdb')
        assert_greater(len(x), 0, "This test needs non empty data.")
        for i in range(len(x))[::-1]:
            if os.path.isdir(path_dst):
                shutil.rmtree(path_dst)
            c.copy_samples_lmdb(path_src, path_dst, [i])
            assert_true(os.path.isdir(path_dst), "failed to save LMDB for i=%d" % (i,))

            y = r.read_values(path_dst)
            assert_equal(len(y), 1, "Single element expected.")
            assert_true(np.all(x[i][0] == y[0][0]), "Wrong content copied.")
            assert_true(np.all(x[i][1] == y[0][1]), "Wrong content copied.")
Example #5
0
    def test_copy_samples_no_append(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset')
        c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
        c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
        c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
        c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
        assert_true(os.path.isdir(path_dst), "failed to save LMDB")

        y = r.read_values(path_dst)
        assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
        for a, b in zip(x[0::2], y):  # skip element in x
            assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
            assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
Example #6
0
 def test_read_values_float(self, mock_dat):
     
     img2_data = self.img1_data[2:,1:,:]
     
     # mocks
     df = [self.img1_data.astype(float).flatten().ravel().tolist(),
           img2_data.astype(float).flatten().ravel().tolist()]
     
     mock_dat.return_value.ParseFromString.return_value = ""
     type(mock_dat.return_value).data = PropertyMock(return_value='')
     type(mock_dat.return_value).float_data = PropertyMock(side_effect=df)
     type(mock_dat.return_value).channels = PropertyMock(side_effect=[4, 2])
     type(mock_dat.return_value).height = PropertyMock(side_effect=[2, 1])
     type(mock_dat.return_value).width = PropertyMock(side_effect=[3, 3])
     
     v = r.read_values(self.path_lmdb, float)
     
     assert_equal(len(v), 2, "Unexpected no. of elements.")
     
     x, _ = v[0]
     for ch in range(4):
         for row in range(2):
             for col in range(3):
                 assert_equal(x[ch, row, col], self.img1_data[ch, row, col])
     
     x, _ = v[1]
     
     for ch in range(2):
         for row in range(1):
             for col in range(3):
                 assert_equal(x[ch, row, col], img2_data[ch, row, col])
Example #7
0
    def test_read_values_pixels_with_labels_uint8(self, mock_dat):
        
        # mocks
        dstr = ['\x01\x04\x07\n\r\x10\x13\x16\x02\x05\x08\x0b\x0e\x11\x14\x17\x03\x06\t\x0c\x0f\x12\x15\x18',
                '\x10\x16\x11\x17\x12\x18']
        
        mock_dat.return_value.ParseFromString.return_value = ""
        type(mock_dat.return_value).data = PropertyMock(side_effect=dstr)
        type(mock_dat.return_value).channels = PropertyMock(side_effect=[3, 3])
        type(mock_dat.return_value).height = PropertyMock(side_effect=[4, 2])
        type(mock_dat.return_value).width = PropertyMock(side_effect=[2, 1])
        type(mock_dat.return_value).label = PropertyMock(side_effect=[1, 0])
        
        v = r.read_values(self.path_lmdb, np.uint8)
        
        assert_equal(len(v), 2)
        
        x, l = v[0]
        for ch in range(3):
            for row in range(4):
                for col in range(2):
                    assert_equal(x[ch, row, col], self.img1_data[row, col, ch])

        assert_equal(l, 1, "Unexpected 1st label")
        
        x, l = v[1]
        img2_data = self.img1_data[2:,1:,:]
        
        for ch in range(3):
            for row in range(2):
                for col in range(1):
                    assert_equal(x[ch, row, col], img2_data[row, col, ch])
                    
        assert_equal(l, 0, "Unexpected 2nd label")
Example #8
0
    def test_copy_samples_subset(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        assert_greater(len(x), 0, "This test needs non empty data.")
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset')
        keys = range(0, len(x), 2)
        assert_greater(len(keys), 0, "This test needs a non-empty subset.")
        assert_greater(len(x), len(keys), "Need subset, not all elements.")
        c.copy_samples_lmdb(path_src, path_dst, keys)
        assert_true(os.path.isdir(path_dst), "failed to save LMDB")

        y = r.read_values(path_dst)
        assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
        for (x_val, x_label), (y_val, y_label) in zip(x[0::2], y):  # skip element in x
            assert_true(np.all(x_val == y_val), "Wrong content copied.")
            assert_true(np.all(x_label == y_label), "Wrong content copied.")
Example #9
0
    def test_concatenate(self):

        path_src0 = os.path.join(self.dir_tmp, 'x0_lmdb')
        x0 = r.read_values(path_src0)
        path_src1 = os.path.join(self.dir_tmp, 'x1_lmdb')
        x1 = r.read_values(path_src1)
        path_src2 = os.path.join(self.dir_tmp, 'x2_lmdb')
        x2 = r.read_values(path_src2)
        path_dst = os.path.join(self.dir_tmp, 'test_concatenate_lmdb')
        c.concatenate_lmdb([path_src0, path_src1, path_src2], path_dst)
        assert_true(os.path.isdir(path_dst), "failed to save LMDB")

        y = r.read_values(path_dst)
        assert_equal(len(x0) + len(x1) + len(x2), len(y), "Wrong number of elements copied.")
        x = x0
        x.extend(x1)
        x.extend(x2)
        assert_equal(len(x), len(y), "Wrong number of elements copied.")
        for a, b in zip(x, y):
            assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
            assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
Example #10
0
    def test_copy_samples_single_with_data_func(self):

        path_src = os.path.join(self.dir_tmp, 'x_lmdb')
        x = r.read_values(path_src)
        path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_single_with_data_func_lmdb')
        assert_greater(len(x), 0, "This test needs non empty data.")
        ALPHA = -10  # non-zero
        def func_data_mul(value):
            _, v = r.unpack_raw_datum(value)
            dat = caffe.io.array_to_datum(ALPHA * v, int(v.flatten()[0]))
            return dat.SerializeToString()

        for i in xrange(len(x)):
            if os.path.isdir(path_dst):
                shutil.rmtree(path_dst)
            c.copy_samples_lmdb(path_src, path_dst, [i], func_data=func_data_mul)
            assert_true(os.path.isdir(path_dst), "failed to save LMDB for i=%d" % (i,))

            y = r.read_values(path_dst)
            assert_equal(len(y), 1, "Single element expected.")
            assert_true(np.all(x[i][0] * ALPHA == y[0][0]), "Wrong content copied.")
            assert_true(np.all(int(x[i][0].flatten()[0]) == y[0][1]), "Wrong content copied for label.")
Example #11
0
    def test_read_values_pixels_no_labels(self, mock_dat):

        # expected content: img1_data, img1_data[2:,1:,:]

        # mocks
        dstr = [
            '\x01\x04\x07\n\r\x10\x13\x16\x02\x05\x08\x0b\x0e\x11\x14\x17\x03\x06\t\x0c\x0f\x12\x15\x18',
            '\x10\x16\x11\x17\x12\x18'
        ]

        mock_dat.return_value.ParseFromString.return_value = ""
        type(mock_dat.return_value).data = PropertyMock(side_effect=dstr)
        type(mock_dat.return_value).channels = PropertyMock(side_effect=[3, 3])
        type(mock_dat.return_value).height = PropertyMock(side_effect=[4, 2])
        type(mock_dat.return_value).width = PropertyMock(side_effect=[2, 1])
        type(mock_dat.return_value).label = PropertyMock(return_value=0)

        v = r.read_values(self.path_lmdb)

        assert_equal(len(v), 2)

        x, l = v[0]
        for ch in range(3):
            for row in range(4):
                for col in range(2):
                    assert_equal(x[ch, row, col], self.img1_data[row, col, ch])

        assert_equal(l, 0, "Unexpected 1st label")

        x, l = v[1]
        img2_data = self.img1_data[2:, 1:, :]

        for ch in range(3):
            for row in range(2):
                for col in range(1):
                    assert_equal(x[ch, row, col], img2_data[row, col, ch])

        assert_equal(l, 0, "Unexpected 2nd label")