def check_suilio_ids(inname):
print('ID num of SULIO:', inname)
ioin = sulio.DataFrame(inname, debug=True)
_, header0 = ioin.read(0)
print(header0[1] - header0[0])
imgnum_total = 0
for idd in range(header0[0], header0[1]):
_, header = ioin.read(idd)
imgnum = header[1] - header[0]
imgnum_total += imgnum
print('Image num:', imgnum_total)
def push_sulio(self, inname):
print('Pushing SULIO:', inname)
ioin = sulio.DataFrame(inname, debug=True)
_, header0 = ioin.read(0)
for idd in tqdm(range(header0[0], header0[1])):
_, header = ioin.read(idd)
meta = [self.pos]
for idx in range(header[0], header[1]):
img = ioin.read_data(idx)
self.ioout.write_idx(self.pos, img)
self.pos += 1
meta.append(self.pos)
self.class_metas.append(meta)
def __init__(self, outname, listfile):
self.list = pickle.load(open(listfile, 'rb'))
self.ioout = sulio.DataFrame(outname, 'w', debug=True)
self.class_metas = []
self.pos = 1
self.info = []
for item in self.list:
t = item['type']
if t == 'sulio':
self.push_sulio(item['data'])
elif t == 'mxrec':
self.push_mxrecord(item['data'])
elif t == 'folder':
self.push_folder(item['data'])
self.info.append({'class_len': len(self.class_metas)})
pickle.dump(self.info, open(os.path.join(outname, 'info.pkl'), 'wb'))
shutil.copy(listfile, os.path.join(outname, 'pathinfo.pkl'))
def __init__(self):
self.ioout = sulio.DataFrame('/data/emore_exp/data_emore_sul/',
debug=True)
self.idx, self.max_label = self.get_data()
print('Sample num:', len(self.idx))
import numpy as np
import cv2
from TorchSUL import DataReader
import random
from TorchSUL import sulio
ioout = sulio.DataFrame('../data_emore_sul/', debug=True)
block_size = 32
stride = 16
n_grid = 6
def adjust_img(img):
# img = img[:16*4]
if random.random()>0.5:
img = np.flip(img, axis=1)
img = np.float32(img)
img = img / 127.5 - 1.
img = np.transpose(img, (2,0,1))
# res = []
# for i in range(n_grid):
# for j in range(n_grid):
# start_h = i*stride
# start_w = j*stride
# res.append(img[:,start_h:start_h+block_size, start_w:start_w+block_size])
# res = np.float32(res)
res = img
return res
def pre_process(inp):
idx, labels = list(zip(*inp))
frames = []
def __init__(self, outname, max_id):
self.ioout = sulio.DataFrame(outname, 'w', debug=True)
self.class_metas = []
self.pos = 1
def __init__(self):
self.ioout = sulio.DataFrame('../500k_cleanv3/500kfull_v3_02/',
debug=True)
self.idx, self.max_label = self.get_data()
import numpy as np
import cv2
from TorchSUL import DataReader
import random
from TorchSUL import sulio
ioout = sulio.DataFrame('../../face_data/part2/', debug=True)
def adjust_img(img):
# img = img[:16*4]
if random.random() > 0.5:
img = np.flip(img, axis=1)
img = np.float32(img)
# img = img[:64]
img = img / 127.5 - 1.
img = np.transpose(img, (2, 0, 1))
return img
def pre_process(inp):
idx, labels = list(zip(*inp))
frames = []
for i in idx:
# print(i)
img = ioout.read_data(i)
frames.append(img)
# print(type(img))
pack = list(zip(*[frames, labels]))
return pack