def check_suilio_ids(inname): print('ID num of SULIO:', inname) ioin = sulio.DataFrame(inname, debug=True) _, header0 = ioin.read(0) print(header0[1] - header0[0]) imgnum_total = 0 for idd in range(header0[0], header0[1]): _, header = ioin.read(idd) imgnum = header[1] - header[0] imgnum_total += imgnum print('Image num:', imgnum_total)
def push_sulio(self, inname): print('Pushing SULIO:', inname) ioin = sulio.DataFrame(inname, debug=True) _, header0 = ioin.read(0) for idd in tqdm(range(header0[0], header0[1])): _, header = ioin.read(idd) meta = [self.pos] for idx in range(header[0], header[1]): img = ioin.read_data(idx) self.ioout.write_idx(self.pos, img) self.pos += 1 meta.append(self.pos) self.class_metas.append(meta)
def __init__(self, outname, listfile): self.list = pickle.load(open(listfile, 'rb')) self.ioout = sulio.DataFrame(outname, 'w', debug=True) self.class_metas = [] self.pos = 1 self.info = [] for item in self.list: t = item['type'] if t == 'sulio': self.push_sulio(item['data']) elif t == 'mxrec': self.push_mxrecord(item['data']) elif t == 'folder': self.push_folder(item['data']) self.info.append({'class_len': len(self.class_metas)}) pickle.dump(self.info, open(os.path.join(outname, 'info.pkl'), 'wb')) shutil.copy(listfile, os.path.join(outname, 'pathinfo.pkl'))
def __init__(self): self.ioout = sulio.DataFrame('/data/emore_exp/data_emore_sul/', debug=True) self.idx, self.max_label = self.get_data() print('Sample num:', len(self.idx))
import numpy as np import cv2 from TorchSUL import DataReader import random from TorchSUL import sulio ioout = sulio.DataFrame('../data_emore_sul/', debug=True) block_size = 32 stride = 16 n_grid = 6 def adjust_img(img): # img = img[:16*4] if random.random()>0.5: img = np.flip(img, axis=1) img = np.float32(img) img = img / 127.5 - 1. img = np.transpose(img, (2,0,1)) # res = [] # for i in range(n_grid): # for j in range(n_grid): # start_h = i*stride # start_w = j*stride # res.append(img[:,start_h:start_h+block_size, start_w:start_w+block_size]) # res = np.float32(res) res = img return res def pre_process(inp): idx, labels = list(zip(*inp)) frames = []
def __init__(self, outname, max_id): self.ioout = sulio.DataFrame(outname, 'w', debug=True) self.class_metas = [] self.pos = 1
def __init__(self): self.ioout = sulio.DataFrame('../500k_cleanv3/500kfull_v3_02/', debug=True) self.idx, self.max_label = self.get_data()
import numpy as np import cv2 from TorchSUL import DataReader import random from TorchSUL import sulio ioout = sulio.DataFrame('../../face_data/part2/', debug=True) def adjust_img(img): # img = img[:16*4] if random.random() > 0.5: img = np.flip(img, axis=1) img = np.float32(img) # img = img[:64] img = img / 127.5 - 1. img = np.transpose(img, (2, 0, 1)) return img def pre_process(inp): idx, labels = list(zip(*inp)) frames = [] for i in idx: # print(i) img = ioout.read_data(i) frames.append(img) # print(type(img)) pack = list(zip(*[frames, labels])) return pack