def load_bin(path, rootdir, image_size=[112, 112]):
align5p = []
if not os.path.exists(rootdir):
os.mkdir(rootdir)
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]],
dtype=np.float32,
rootdir=rootdir,
mode='w')
imgDir = os.path.join(rootdir, 'imgs')
if not os.path.exists(imgDir):
os.mkdir(imgDir)
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(imgDir, '{}.jpg'.format(i)), img,
[int(cv2.IMWRITE_JPEG_QUALITY), 95])
align5p.append(os.path.join(imgDir, '{}.jpg'.format(i)))
i += 1
if i % 1000 == 0:
print('loading bin', i)
print(data.shape)
np.save(os.path.join(rootdir,
rootdir.split('/')[-1] + '_list'),
np.array(issame_list))
np.save(os.path.join('../processed/', 'list.npy'), np.array(issame_list))
np.save(os.path.join(rootdir, 'list.npy'), np.array(issame_list))
print(os.path.join(rootdir, 'align5p.npy'))
np.save(os.path.join(rootdir, 'align5p.npy'), align5p)
return data, issame_list
def load_bin(path, rootdir, transform, image_size=[112, 112]):
if not rootdir.exists():
rootdir.mkdir()
# path = str(path)
# rootdir = str(rootdir)
bins, issame_list = pickle.load(open(path, "rb"), encoding="bytes")
# bins, issame_list = pickle.load(open(path, 'rb'))
data = bcolz.fill(
[len(bins), 3, image_size[0], image_size[1]],
dtype=np.float32,
rootdir=rootdir,
mode="w",
)
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = transform(img)
i += 1
if i % 1000 == 0:
print("loading bin", i)
print(data.shape)
np.save(str(rootdir) + "_list", np.array(issame_list))
return data, issame_list
def load_noonan_val_pair(path, rootdir, transform, image_size=[112,112]):
# path: path for original images, str
# rootdir: path to store images and issame_list, Path
mtcnn = MTCNN()
print('mtcnn loaded')
if not rootdir.exists():
rootdir.mkdir()
images = os.listdir(path)
comb_size = len(images) * (len(images) - 1) / 2
pairs = combinations(images, 2)
data = bcolz.fill([int(comb_size * 2), 3, image_size[0], image_size[1]], dtype=np.float32, rootdir=rootdir, mode='w')
issame_list = np.zeros(int(comb_size))
i = 0
for pair in pairs:
img0 = Image.open(os.path.join(path, pair[0]))
if img0.size != (112, 112):
img0 = mtcnn.align(img0)
img1 = Image.open(os.path.join(path, pair[1]))
if img1.size != (112, 112):
img1 = mtcnn.align(img1)
data[2*i, ...] = transform(img0)
data[2*i + 1, ...] = transform(img1)
if ('noonan' in pair[0] and 'noonan' in pair[1]) or ('normal' in pair[0] and 'normal' in pair[1]):
issame_list[i] = 1
i += 1
if i % 1000 == 0:
print('loading noonan', i)
print(data.shape)
np.save(str(rootdir)+'_list', np.array(issame_list))
return data, issame_list
def test02(self):
"""Testing `append()` method (several rows)"""
a = np.ones((4, 3), dtype="i4") * 3
b = bcolz.fill((1, 3), 3, dtype="i4", rootdir=self.rootdir)
b.append([(3, 3, 3)] * 3)
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def load_bin(path, rootdir, transform, image_size=[112, 112]):
"""
:param path: dove saranno salvate le immagini
:param rootdir: path dataset immagini
:param transform: trasformazioni applicate alle immagini
:param image_size: dimensione immagine di input
:return:
"""
if not rootdir.exists():
rootdir.mkdir()
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]],
dtype=np.float32,
rootdir=rootdir,
mode='w')
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = transform(img)
i += 1
if i % 1000 == 0:
print('loading bin', i)
print(data.shape)
np.save(str(rootdir) + '_list', np.array(issame_list))
return data, issame_list
def test00b(self):
"""Testing `append()` method (correct shape, single row)"""
a = np.ones((2, 300), dtype="i4") * 3
b = bcolz.fill((1, 300), 3, dtype="i4", rootdir=self.rootdir)
b.append((3,) * 300)
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def test03a(self):
"""Testing `fill` constructor (scalar default)"""
a = np.ones((2, 200), dtype='(4,)i4') * 3
b = bcolz.fill((2, 200), 3, dtype='(4,)i4', rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
# print "b->", `b`
assert_array_equal(a, b, "Arrays are not equal")
def test00b(self):
"""Testing `__getitem()__` method with only a start (slice)"""
a = np.ones((27, 2700), dtype="i4") * 3
b = bcolz.fill((27, 2700), 3, dtype="i4", rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
sl = slice(1)
self.assertTrue(a[sl].shape == b[sl].shape, "Shape is not equal")
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test02(self):
"""Testing `__getitem()__` method with a start, stop, step"""
a = np.ones((10, 2), dtype="i4") * 3
b = bcolz.fill((10, 2), 3, dtype="i4", rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
sl = slice(1, 9, 2)
# print "b[sl]->", `b[sl]`
self.assertTrue(a[sl].shape == b[sl].shape, "Shape is not equal")
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test04(self):
"""Testing `fill` constructor with open and resize (array default)"""
a = np.ones((3,200), dtype='(4,)i4')*3
b = bcolz.fill((2,200), [3,3,3,3], dtype='(4,)i4', rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
c = np.ones((1,200), dtype='(4,)i4')*3
b.append(c)
#print "b->", `b`, len(b), b[1]
assert_array_equal(a, b, "Arrays are not equal")
def test04(self):
"""Testing `fill` constructor with open and resize (array default)"""
a = np.ones((3, 200), dtype='(4,)i4') * 3
b = bcolz.fill((2, 200), 3, dtype='(4,)i4', rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
c = np.ones((1, 200), dtype='(4,)i4') * 3
b.append(c)
# print "b->", `b`, len(b), b[1]
assert_array_equal(a, b, "Arrays are not equal")
def test00a(self):
"""Testing `__getitem()__` method with only a start (scalar)"""
a = np.ones((2, 3), dtype="i4") * 3
b = bcolz.fill((2, 3), 3, dtype="i4", rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
sl = 1
# print "b[sl]->", `b[sl]`
self.assertTrue(a[sl].shape == b[sl].shape, "Shape is not equal")
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test05(self):
"""Testing `fill` constructor with open and resize (nchunks>1)"""
a = np.ones((3,2000), dtype='(4,)i4')*3
b = bcolz.fill((2,2000), [3,3,3,3], dtype='(4,)i4', rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
c = np.ones((1,2000), dtype='(4,)i4')*3
b.append(c)
#print "b->", `b`
# We need to use the b[:] here to overcome a problem with the
# assert_array_equal() function
assert_array_equal(a, b[:], "Arrays are not equal")
def test05(self):
"""Testing `fill` constructor with open and resize (nchunks>1)"""
a = np.ones((3, 2000), dtype='(4,)i4') * 3
b = bcolz.fill((2, 2000), 3, dtype='(4,)i4', rootdir=self.rootdir)
if self.open:
b = bcolz.open(rootdir=self.rootdir)
c = np.ones((1, 2000), dtype='(4,)i4') * 3
b.append(c)
# print "b->", `b`
# We need to use the b[:] here to overcome a problem with the
# assert_array_equal() function
assert_array_equal(a, b[:], "Arrays are not equal")
def test02b(self):
"""Testing `__setitem()__` method with start,stop,step (scalar)"""
a = np.ones((10, 2), dtype="i4") * 3
b = bcolz.fill((10, 2), 3, dtype="i4", rootdir=self.rootdir)
sl = slice(1, 8, 3)
a[sl, :] = range(2)
b[sl] = range(2)
if self.open:
b.flush()
b = bcolz.open(rootdir=self.rootdir)
# print "b[sl]->", `b[sl]`, `b`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test00b(self):
"""Testing `__setitem()__` method with only a start (vector)"""
a = np.ones((200, 300), dtype="i4") * 3
b = bcolz.fill((200, 300), 3, dtype="i4", rootdir=self.rootdir)
sl = slice(1)
a[sl, :] = range(300)
b[sl] = range(300)
if self.open:
b.flush()
b = bcolz.open(rootdir=self.rootdir)
# print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def test01a(self):
"""Testing `__setitem()__` method with start,stop (scalar)"""
a = np.ones((500, 200), dtype="i4") * 3
b = bcolz.fill((500, 200), 3, dtype="i4", rootdir=self.rootdir,
cparams=bcolz.cparams())
sl = slice(100, 400)
a[sl, :] = 0
b[sl] = 0
if self.open:
b.flush()
b = bcolz.open(rootdir=self.rootdir)
# print "b[sl]->", `b[sl]`
assert_array_equal(a[sl], b[sl], "Arrays are not equal")
def convert_image_to_bcolz(pair_filename,
image_dir,
save_dir,
input_size=[112, 112]):
from torchvision import transforms as trans
import bcolz
transform = trans.Compose(
[trans.ToTensor(),
trans.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])
if not os.path.exists(save_dir):
os.makedirs(save_dir)
faces_list1, faces_list2, issames_data = read_pair_data(pair_filename)
print("have {} pair".format(len(issames_data)))
print("have {} pair".format(len(faces_list1)))
issames_data = np.array(issames_data)
issames_data = np.where(issames_data > 0, True, False)
data = bcolz.fill(shape=[
len(faces_list1) + len(faces_list2), 3, input_size[0], input_size[1]
],
dtype=np.float32,
rootdir=save_dir,
mode='w')
for i, (face1_path, face2_path,
issame) in enumerate(zip(faces_list1, faces_list2, issames_data)):
# pred_id, pred_scores = faceRec.predict(faces)
# 或者使用get_faces_embedding()获得embedding，再比较compare_embedding()
if image_dir:
face1_path = os.path.join(image_dir, face1_path)
face2_path = os.path.join(image_dir, face2_path)
face1 = image_processing.read_image_gbk(face1_path, colorSpace="BGR")
face2 = image_processing.read_image_gbk(face2_path, colorSpace="BGR")
face1 = image_processing.resize_image(face1,
resize_height=input_size[0],
resize_width=input_size[1])
face2 = image_processing.resize_image(face2,
resize_height=input_size[0],
resize_width=input_size[1])
# img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
# image_processing.cv_show_image("image_dict",img)
face1 = Image.fromarray(face1.astype(np.uint8))
face2 = Image.fromarray(face2.astype(np.uint8))
data[i * 2, ...] = transform(face1)
data[i * 2 + 1, ...] = transform(face2)
if i % 100 == 0:
print('loading bin', i)
print(data.shape)
np.save(str(save_dir) + '_list', issames_data)
def load_bin(path, rootdir: Path, transform, image_size=[112, 112]):
if not rootdir.exists():
rootdir.mkdir()
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill(shape=[len(bins), 3, image_size[0], image_size[1]],
dtype=np.float32,
rootdir=rootdir,
mode='w')
for i in range(len(bins)):
img = mx.image.imdecode(bins[i]).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = transform(img)
np.save(str(rootdir) + '_list', np.array(issame_list))
def load_bin(path, rootdir, transform, image_size = [112, 112]):
if not rootdir.exists():
rootdir.mkdir()
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]], dtype = np.float32, rootdir = rootdir, mode = 'w')
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = transform(img)
i += 1
if i % 1000 == 0:
print('loading bin', i)
print(data.shape)
np.save(str(rootdir) + '_list', np.array(issame_list))
return data, issame_list
def load_bin(path, rootdir, transform, image_size=[112,112]):
if not Path(rootdir).exists():
Path(rootdir).mkdir()
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]], dtype=np.float32, rootdir=(rootdir), mode='w')
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = transform(img)
i += 1
if i % 1000 == 0:
print('loading bin', i)
print(data.shape)
np.save(str(Path(rootdir))+'_list', np.array(issame_list))
return data, issame_list
def load_bin(path, image_size=[112, 112]):
rootdir = path[:-4]
bins, issame_list = pickle.load(open(path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]],
dtype=np.float32, rootdir=rootdir, mode='w')
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = Image.fromarray(img.astype(np.uint8))
data[i, ...] = TRANSFORM(img)
if i % 1000 == 0:
print('Loading bin...', i)
print(data.shape)
np.save(str(rootdir) + '_list', np.array(issame_list))
return data, issame_list
def load_bin(bin_path, out_path, image_size = [112, 112]):
bins, issame_list = pickle.load(open(bin_path, 'rb'), encoding='bytes')
data = bcolz.fill([len(bins), 3, image_size[0], image_size[1]], dtype = np.float32, rootdir = out_path, mode = 'w')
for i in range(len(bins)):
_bin = bins[i]
img = mx.image.imdecode(_bin).asnumpy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
img = cv2.resize(img, dsize=(image_size[0], image_size[1]),)
data[i, ...] = img.reshape(3, image_size[0], image_size[1])
i += 1
if i % 1000 == 0:
print('loading bin', i)
print(data.shape)
np.save(str(out_path) + '_list', np.array(issame_list))
return data, issame_list
from time import time
import numpy as np
import bcolz
N = 1e8
dtype = "i4"
t0 = time()
a = np.ones(N, dtype=dtype)
print("Time numpy.ones() --> %.4f" % (time() - t0))
t0 = time()
ac = bcolz.fill(N, dtype=dtype, dflt=1)
# ac = bcolz.carray(a)
print("Time carray.fill(dflt=1) --> %.4f" % (time() - t0))
print("ac-->", repr(ac))
t0 = time()
sa = a.sum()
print("Time a.sum() --> %.4f" % (time() - t0))
t0 = time()
sac = ac.sum()
print("Time ac.sum() --> %.4f" % (time() - t0))
assert sa == sac
from time import time
import numpy as np
import bcolz
N = 1e8
dtype = 'i4'
t0 = time()
a = np.ones(N, dtype=dtype)
print("Time numpy.ones() --> %.4f" % (time() - t0))
t0 = time()
ac = bcolz.fill(N, dtype=dtype, dflt=1)
# ac = bcolz.carray(a)
print("Time carray.fill(dflt=1) --> %.4f" % (time() - t0))
print("ac-->", repr(ac))
t0 = time()
sa = a.sum()
print("Time a.sum() --> %.4f" % (time() - t0))
t0 = time()
sac = ac.sum()
print("Time ac.sum() --> %.4f" % (time() - t0))
assert (sa == sac)
def test01(self):
"""Testing `append()` method (incorrect shape)"""
a = np.ones((2, 3), dtype="i4") * 3
b = bcolz.fill((1, 3), 3, dtype="i4", rootdir=self.rootdir)
self.assertRaises(ValueError, b.append, [(3, 3)])