def test_copy_samples_subset_with_data_func(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
assert_greater(len(x), 0, "This test needs non empty data.")
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset_with_data_func_lmdb')
keys = range(0, len(x), 2)
assert_greater(len(keys), 0, "This test needs a non-empty subset.")
assert_greater(len(x), len(keys), "Need subset, not all elements.")
ALPHA = -10 # non-zero
def func_data_mul(value):
_, x = r.unpack_raw_datum(value)
dat = caffe.io.array_to_datum(ALPHA * x, int(x.flatten()[0]))
return dat.SerializeToString()
c.copy_samples_lmdb(path_src, path_dst, keys, func_data=func_data_mul)
assert_true(os.path.isdir(path_dst), "failed to save LMDB")
y = r.read_values(path_dst)
assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
for (x_val, x_label), (y_val, y_label) in zip(x[0::2], y): # skip element in x
assert_true(np.all(x_val * ALPHA == y_val), "Wrong content copied.")
assert_true(np.all(int(x_val.flatten()[0]) == y_label), "Wrong content copied for label.")
assert_false(np.all(x_label == y_label), "Wrong content copied for label.")
def test_copy_samples_all(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_all_lmdb')
c.copy_samples_lmdb(path_src, path_dst, range(len(x)))
assert_true(os.path.isdir(path_dst), "failed to save LMDB")
y = r.read_values(path_dst)
assert_equal(len(x), len(y), "Wrong number of elements copied.")
for a, b in zip(x, y):
assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
def test_read_values_float(self, mock_dat):
img2_data = self.img1_data[2:, 1:, :]
# mocks
df = [
self.img1_data.astype(float).flatten().ravel().tolist(),
img2_data.astype(float).flatten().ravel().tolist()
]
mock_dat.return_value.ParseFromString.return_value = ""
type(mock_dat.return_value).data = PropertyMock(return_value='')
type(mock_dat.return_value).float_data = PropertyMock(side_effect=df)
type(mock_dat.return_value).channels = PropertyMock(side_effect=[4, 2])
type(mock_dat.return_value).height = PropertyMock(side_effect=[2, 1])
type(mock_dat.return_value).width = PropertyMock(side_effect=[3, 3])
v = r.read_values(self.path_lmdb, float)
assert_equal(len(v), 2, "Unexpected no. of elements.")
x, _ = v[0]
for ch in range(4):
for row in range(2):
for col in range(3):
assert_equal(x[ch, row, col], self.img1_data[ch, row, col])
x, _ = v[1]
for ch in range(2):
for row in range(1):
for col in range(3):
assert_equal(x[ch, row, col], img2_data[ch, row, col])
def test_copy_samples_single_reverse(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_single_lmdb')
assert_greater(len(x), 0, "This test needs non empty data.")
for i in range(len(x))[::-1]:
if os.path.isdir(path_dst):
shutil.rmtree(path_dst)
c.copy_samples_lmdb(path_src, path_dst, [i])
assert_true(os.path.isdir(path_dst), "failed to save LMDB for i=%d" % (i,))
y = r.read_values(path_dst)
assert_equal(len(y), 1, "Single element expected.")
assert_true(np.all(x[i][0] == y[0][0]), "Wrong content copied.")
assert_true(np.all(x[i][1] == y[0][1]), "Wrong content copied.")
def test_copy_samples_no_append(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset')
c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
c.copy_samples_lmdb(path_src, path_dst, range(0, len(x), 2))
assert_true(os.path.isdir(path_dst), "failed to save LMDB")
y = r.read_values(path_dst)
assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
for a, b in zip(x[0::2], y): # skip element in x
assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
def test_read_values_float(self, mock_dat):
img2_data = self.img1_data[2:,1:,:]
# mocks
df = [self.img1_data.astype(float).flatten().ravel().tolist(),
img2_data.astype(float).flatten().ravel().tolist()]
mock_dat.return_value.ParseFromString.return_value = ""
type(mock_dat.return_value).data = PropertyMock(return_value='')
type(mock_dat.return_value).float_data = PropertyMock(side_effect=df)
type(mock_dat.return_value).channels = PropertyMock(side_effect=[4, 2])
type(mock_dat.return_value).height = PropertyMock(side_effect=[2, 1])
type(mock_dat.return_value).width = PropertyMock(side_effect=[3, 3])
v = r.read_values(self.path_lmdb, float)
assert_equal(len(v), 2, "Unexpected no. of elements.")
x, _ = v[0]
for ch in range(4):
for row in range(2):
for col in range(3):
assert_equal(x[ch, row, col], self.img1_data[ch, row, col])
x, _ = v[1]
for ch in range(2):
for row in range(1):
for col in range(3):
assert_equal(x[ch, row, col], img2_data[ch, row, col])
def test_read_values_pixels_with_labels_uint8(self, mock_dat):
# mocks
dstr = ['\x01\x04\x07\n\r\x10\x13\x16\x02\x05\x08\x0b\x0e\x11\x14\x17\x03\x06\t\x0c\x0f\x12\x15\x18',
'\x10\x16\x11\x17\x12\x18']
mock_dat.return_value.ParseFromString.return_value = ""
type(mock_dat.return_value).data = PropertyMock(side_effect=dstr)
type(mock_dat.return_value).channels = PropertyMock(side_effect=[3, 3])
type(mock_dat.return_value).height = PropertyMock(side_effect=[4, 2])
type(mock_dat.return_value).width = PropertyMock(side_effect=[2, 1])
type(mock_dat.return_value).label = PropertyMock(side_effect=[1, 0])
v = r.read_values(self.path_lmdb, np.uint8)
assert_equal(len(v), 2)
x, l = v[0]
for ch in range(3):
for row in range(4):
for col in range(2):
assert_equal(x[ch, row, col], self.img1_data[row, col, ch])
assert_equal(l, 1, "Unexpected 1st label")
x, l = v[1]
img2_data = self.img1_data[2:,1:,:]
for ch in range(3):
for row in range(2):
for col in range(1):
assert_equal(x[ch, row, col], img2_data[row, col, ch])
assert_equal(l, 0, "Unexpected 2nd label")
def test_copy_samples_subset(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
assert_greater(len(x), 0, "This test needs non empty data.")
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_subset')
keys = range(0, len(x), 2)
assert_greater(len(keys), 0, "This test needs a non-empty subset.")
assert_greater(len(x), len(keys), "Need subset, not all elements.")
c.copy_samples_lmdb(path_src, path_dst, keys)
assert_true(os.path.isdir(path_dst), "failed to save LMDB")
y = r.read_values(path_dst)
assert_equal(int(len(x) / 2), len(y), "Wrong number of elements copied.")
for (x_val, x_label), (y_val, y_label) in zip(x[0::2], y): # skip element in x
assert_true(np.all(x_val == y_val), "Wrong content copied.")
assert_true(np.all(x_label == y_label), "Wrong content copied.")
def test_concatenate(self):
path_src0 = os.path.join(self.dir_tmp, 'x0_lmdb')
x0 = r.read_values(path_src0)
path_src1 = os.path.join(self.dir_tmp, 'x1_lmdb')
x1 = r.read_values(path_src1)
path_src2 = os.path.join(self.dir_tmp, 'x2_lmdb')
x2 = r.read_values(path_src2)
path_dst = os.path.join(self.dir_tmp, 'test_concatenate_lmdb')
c.concatenate_lmdb([path_src0, path_src1, path_src2], path_dst)
assert_true(os.path.isdir(path_dst), "failed to save LMDB")
y = r.read_values(path_dst)
assert_equal(len(x0) + len(x1) + len(x2), len(y), "Wrong number of elements copied.")
x = x0
x.extend(x1)
x.extend(x2)
assert_equal(len(x), len(y), "Wrong number of elements copied.")
for a, b in zip(x, y):
assert_true(np.all(a[0] == b[0]), "Wrong content copied.")
assert_true(np.all(a[1] == b[1]), "Wrong content copied.")
def test_copy_samples_single_with_data_func(self):
path_src = os.path.join(self.dir_tmp, 'x_lmdb')
x = r.read_values(path_src)
path_dst = os.path.join(self.dir_tmp, 'test_copy_samples_single_with_data_func_lmdb')
assert_greater(len(x), 0, "This test needs non empty data.")
ALPHA = -10 # non-zero
def func_data_mul(value):
_, v = r.unpack_raw_datum(value)
dat = caffe.io.array_to_datum(ALPHA * v, int(v.flatten()[0]))
return dat.SerializeToString()
for i in xrange(len(x)):
if os.path.isdir(path_dst):
shutil.rmtree(path_dst)
c.copy_samples_lmdb(path_src, path_dst, [i], func_data=func_data_mul)
assert_true(os.path.isdir(path_dst), "failed to save LMDB for i=%d" % (i,))
y = r.read_values(path_dst)
assert_equal(len(y), 1, "Single element expected.")
assert_true(np.all(x[i][0] * ALPHA == y[0][0]), "Wrong content copied.")
assert_true(np.all(int(x[i][0].flatten()[0]) == y[0][1]), "Wrong content copied for label.")
def test_read_values_pixels_no_labels(self, mock_dat):
# expected content: img1_data, img1_data[2:,1:,:]
# mocks
dstr = [
'\x01\x04\x07\n\r\x10\x13\x16\x02\x05\x08\x0b\x0e\x11\x14\x17\x03\x06\t\x0c\x0f\x12\x15\x18',
'\x10\x16\x11\x17\x12\x18'
]
mock_dat.return_value.ParseFromString.return_value = ""
type(mock_dat.return_value).data = PropertyMock(side_effect=dstr)
type(mock_dat.return_value).channels = PropertyMock(side_effect=[3, 3])
type(mock_dat.return_value).height = PropertyMock(side_effect=[4, 2])
type(mock_dat.return_value).width = PropertyMock(side_effect=[2, 1])
type(mock_dat.return_value).label = PropertyMock(return_value=0)
v = r.read_values(self.path_lmdb)
assert_equal(len(v), 2)
x, l = v[0]
for ch in range(3):
for row in range(4):
for col in range(2):
assert_equal(x[ch, row, col], self.img1_data[row, col, ch])
assert_equal(l, 0, "Unexpected 1st label")
x, l = v[1]
img2_data = self.img1_data[2:, 1:, :]
for ch in range(3):
for row in range(2):
for col in range(1):
assert_equal(x[ch, row, col], img2_data[row, col, ch])
assert_equal(l, 0, "Unexpected 2nd label")