def run(self): import imgworker bsz = self.ds.batch_size b_idx = self.ds.macro_decode_buf_idx jdict = self.ds.get_macro_batch() betype = self.ds.backend_type # This macrobatch could be smaller than macro_size for last macrobatch mac_sz = len(jdict['data']) self.ds.tgt_macro[b_idx] = \ jdict['targets'] if 'targets' in jdict else None lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list} img_macro = np.zeros((self.ds.macro_size, self.ds.npixels), dtype=np.uint8) do_center = self.ds.predict or not self.ds.dotransforms do_flip = self.ds.dotransforms imgworker.decode_list(jpglist=jdict['data'], tgt=img_macro[:mac_sz], orig_size=self.ds.output_image_size, crop_size=self.ds.cropped_image_size, center=do_center, flip=do_flip, rgb=self.ds.rgb, nthreads=self.ds.num_workers) if mac_sz < self.ds.macro_size: img_macro[mac_sz:] = 0 # Leave behind the partial minibatch self.ds.minis_per_macro[b_idx] = mac_sz / bsz self.ds.lbl_one_hot[b_idx] = \ {lbl: [None for mini_idx in range(self.ds.minis_per_macro[b_idx])] for lbl in self.ds.label_list} self.ds.img_mini_T[b_idx] = \ [None for mini_idx in range(self.ds.minis_per_macro[b_idx])] for mini_idx in range(self.ds.minis_per_macro[b_idx]): s_idx = mini_idx * bsz e_idx = (mini_idx + 1) * bsz self.ds.img_mini_T[b_idx][mini_idx] = \ img_macro[s_idx:e_idx].T.astype(betype, order='C') if self.ds.img_mini_T[b_idx][mini_idx].shape[1] < bsz: tmp = self.ds.img_mini_T[b_idx][mini_idx].shape[0] mb_residual = self.ds.img_mini_T[b_idx][mini_idx].shape[1] filledbatch = np.vstack((img_macro[s_idx:e_idx], np.zeros((bsz - mb_residual, tmp)))) self.ds.img_mini_T[b_idx][mini_idx] = \ filledbatch.T.astype(betype, order='C') for lbl in self.ds.label_list: hl = np.squeeze(lbl_macro[lbl][s_idx:e_idx]) self.ds.lbl_one_hot[b_idx][lbl][mini_idx] = \ np.eye(self.ds.nclass[lbl])[hl].T.astype(betype, order='C') return
def run(self): import imgworker bsz = self.ds.batch_size b_idx = self.ds.macro_decode_buf_idx jdict = self.ds.get_macro_batch() betype = self.ds.backend_type # This macrobatch could be smaller than macro_size for last macrobatch mac_sz = len(jdict['data']) self.ds.tgt_macro[b_idx] = \ jdict['targets'] if 'targets' in jdict else None lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list} img_macro = np.zeros((self.ds.macro_size, self.ds.npixels), dtype=np.uint8) imgworker.decode_list(jpglist=jdict['data'], tgt=img_macro[:mac_sz], orig_size=self.ds.output_image_size, crop_size=self.ds.cropped_image_size, center=self.ds.predict, flip=True, rgb=self.ds.rgb, nthreads=self.ds.num_workers) if mac_sz < self.ds.macro_size: img_macro[mac_sz:] = 0 # Leave behind the partial minibatch self.ds.minis_per_macro[b_idx] = mac_sz / bsz self.ds.lbl_one_hot[b_idx] = \ {lbl: [None for mini_idx in range(self.ds.minis_per_macro[b_idx])] for lbl in self.ds.label_list} self.ds.img_mini_T[b_idx] = \ [None for mini_idx in range(self.ds.minis_per_macro[b_idx])] for mini_idx in range(self.ds.minis_per_macro[b_idx]): s_idx = mini_idx * bsz e_idx = (mini_idx + 1) * bsz self.ds.img_mini_T[b_idx][mini_idx] = \ img_macro[s_idx:e_idx].T.astype(betype, order='C') if self.ds.img_mini_T[b_idx][mini_idx].shape[1] < bsz: tmp = self.ds.img_mini_T[b_idx][mini_idx].shape[0] mb_residual = self.ds.img_mini_T[b_idx][mini_idx].shape[1] filledbatch = np.vstack((img_macro[s_idx:e_idx], np.zeros((bsz - mb_residual, tmp)))) self.ds.img_mini_T[b_idx][mini_idx] = \ filledbatch.T.astype(betype, order='C') for lbl in self.ds.label_list: hl = np.squeeze(lbl_macro[lbl][s_idx:e_idx]) self.ds.lbl_one_hot[b_idx][lbl][mini_idx] = \ np.eye(self.ds.nclass[lbl])[hl].T.astype(betype, order='C') return
def run(self): import imgworker bsz = self.ds.batch_size batch_idx = self.ds.macro_decode_buf_idx jdict = self.ds.get_macro_batch() ibetype = self.ds.img_dtype # This macrobatch could be smaller than macro_size for last macrobatch macro_size = len(jdict['data']) self.macro_size = macro_size if self.ds.shuffle_macro: shuff_idx = range(macro_size) shuffle(shuff_idx) jdict['data'] = [jdict['data'][i] for i in shuff_idx] for k in self.ds.label_list: jdict['labels'][k] = [jdict['labels'][k][i] for i in shuff_idx] self.ds.target_macro[batch_idx] = \ jdict['targets'] if 'targets' in jdict else None lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list} do_center = self.ds.predict or not self.ds.dotransforms do_flip = self.ds.dotransforms imgworker.decode_list(jpglist=jdict['data'], tgt=self.ds.int_macro[batch_idx][:macro_size], orig_size=self.ds.output_image_size, crop_size=self.ds.cropped_image_size, center=do_center, flip=do_flip, rgb=self.ds.rgb, nthreads=self.ds.num_workers) # This ensures we take the last partial minibatch self.ds.minis_per_macro[batch_idx] = -(-macro_size // bsz) for lbl in self.ds.label_list: hl = np.squeeze(lbl_macro[lbl]) self.ds.lbl_one_hot[batch_idx][lbl][:macro_size] = \ np.eye(self.ds.nclass[lbl], dtype=ibetype)[hl].astype(ibetype, order='C') if macro_size < self.ds.macro_size: self.ds.lbl_one_hot[batch_idx][lbl][macro_size:] = 0 np.subtract(self.ds.int_macro[batch_idx], self.ds.mean_val, self.ds.img_macro[batch_idx]) if macro_size < self.ds.macro_size: self.ds.img_macro[batch_idx][macro_size:] = 0 return
# a[:] = labels numthreads = int(sys.argv[1]) imgsize = 256 innersize = 224 innerpixels = innersize * innersize * 3 startimg = 0 num_imgs = 2048 unpacked = np.zeros((num_imgs, innerpixels), dtype=np.uint8) t0 = time.time() for i in range(24): t1 = time.time() PATH = '/usr/local/data/I1K/macro_batches_256/training_batch_{:d}/'.format( i) fname = os.path.join(PATH, 'training_batch_{:d}.0'.format(i)) tdata = my_unpickle(fname) jpeglist = tdata['data'] # labels = tdata['labels'] iw.decode_list(jpglist=jpeglist[startimg:startimg + num_imgs], tgt=unpacked, orig_size=imgsize, crop_size=innersize, nthreads=numthreads) print("\tC Module version decode time: ", time.time() - t1) print("Total Time:", time.time() - t0) # Now save out an image from the packed array to confirm things are working data = unpacked[5, :].reshape( (3, innersize, innersize)).transpose(1, 2, 0)[:, :, [0, 1, 2]] im = Image.fromarray(data) im.save('out.png')
# labels = [item for sublist in labels for item in sublist] # labels = np.asarray(labels, dtype='float32') # a = np.empty((1,3072), dtype=np.float32) # a[:] = labels numthreads = int(sys.argv[1]) imgsize = 256 innersize = 224 innerpixels = innersize*innersize*3 startimg = 0 num_imgs = 2048 unpacked = np.zeros((num_imgs, innerpixels), dtype=np.uint8) t0 = time.time() for i in range(24): t1 = time.time() PATH = '/usr/local/data/I1K/macro_batches_256/training_batch_{:d}/'.format(i) fname = os.path.join(PATH, 'training_batch_{:d}.0'.format(i)) tdata = my_unpickle(fname) jpeglist = tdata['data'] # labels = tdata['labels'] iw.decode_list(jpglist=jpeglist[startimg:startimg+num_imgs], tgt=unpacked, orig_size=imgsize, crop_size=innersize, nthreads=numthreads) print("\tC Module version decode time: ", time.time() - t1) print("Total Time:", time.time() - t0) # Now save out an image from the packed array to confirm things are working data = unpacked[5,:].reshape( (3, innersize, innersize)).transpose(1,2,0)[:,:,[0,1,2]] im = Image.fromarray(data) im.save('out.png')