def run(self):
import imgworker
bsz = self.ds.batch_size
b_idx = self.ds.macro_decode_buf_idx
jdict = self.ds.get_macro_batch()
betype = self.ds.backend_type
# This macrobatch could be smaller than macro_size for last macrobatch
mac_sz = len(jdict['data'])
self.ds.tgt_macro[b_idx] = \
jdict['targets'] if 'targets' in jdict else None
lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list}
img_macro = np.zeros((self.ds.macro_size, self.ds.npixels),
dtype=np.uint8)
do_center = self.ds.predict or not self.ds.dotransforms
do_flip = self.ds.dotransforms
imgworker.decode_list(jpglist=jdict['data'],
tgt=img_macro[:mac_sz],
orig_size=self.ds.output_image_size,
crop_size=self.ds.cropped_image_size,
center=do_center, flip=do_flip,
rgb=self.ds.rgb,
nthreads=self.ds.num_workers)
if mac_sz < self.ds.macro_size:
img_macro[mac_sz:] = 0
# Leave behind the partial minibatch
self.ds.minis_per_macro[b_idx] = mac_sz / bsz
self.ds.lbl_one_hot[b_idx] = \
{lbl: [None for mini_idx in range(self.ds.minis_per_macro[b_idx])]
for lbl in self.ds.label_list}
self.ds.img_mini_T[b_idx] = \
[None for mini_idx in range(self.ds.minis_per_macro[b_idx])]
for mini_idx in range(self.ds.minis_per_macro[b_idx]):
s_idx = mini_idx * bsz
e_idx = (mini_idx + 1) * bsz
self.ds.img_mini_T[b_idx][mini_idx] = \
img_macro[s_idx:e_idx].T.astype(betype, order='C')
if self.ds.img_mini_T[b_idx][mini_idx].shape[1] < bsz:
tmp = self.ds.img_mini_T[b_idx][mini_idx].shape[0]
mb_residual = self.ds.img_mini_T[b_idx][mini_idx].shape[1]
filledbatch = np.vstack((img_macro[s_idx:e_idx],
np.zeros((bsz - mb_residual, tmp))))
self.ds.img_mini_T[b_idx][mini_idx] = \
filledbatch.T.astype(betype, order='C')
for lbl in self.ds.label_list:
hl = np.squeeze(lbl_macro[lbl][s_idx:e_idx])
self.ds.lbl_one_hot[b_idx][lbl][mini_idx] = \
np.eye(self.ds.nclass[lbl])[hl].T.astype(betype, order='C')
return
def run(self):
import imgworker
bsz = self.ds.batch_size
b_idx = self.ds.macro_decode_buf_idx
jdict = self.ds.get_macro_batch()
betype = self.ds.backend_type
# This macrobatch could be smaller than macro_size for last macrobatch
mac_sz = len(jdict['data'])
self.ds.tgt_macro[b_idx] = \
jdict['targets'] if 'targets' in jdict else None
lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list}
img_macro = np.zeros((self.ds.macro_size, self.ds.npixels),
dtype=np.uint8)
imgworker.decode_list(jpglist=jdict['data'],
tgt=img_macro[:mac_sz],
orig_size=self.ds.output_image_size,
crop_size=self.ds.cropped_image_size,
center=self.ds.predict, flip=True,
rgb=self.ds.rgb,
nthreads=self.ds.num_workers)
if mac_sz < self.ds.macro_size:
img_macro[mac_sz:] = 0
# Leave behind the partial minibatch
self.ds.minis_per_macro[b_idx] = mac_sz / bsz
self.ds.lbl_one_hot[b_idx] = \
{lbl: [None for mini_idx in range(self.ds.minis_per_macro[b_idx])]
for lbl in self.ds.label_list}
self.ds.img_mini_T[b_idx] = \
[None for mini_idx in range(self.ds.minis_per_macro[b_idx])]
for mini_idx in range(self.ds.minis_per_macro[b_idx]):
s_idx = mini_idx * bsz
e_idx = (mini_idx + 1) * bsz
self.ds.img_mini_T[b_idx][mini_idx] = \
img_macro[s_idx:e_idx].T.astype(betype, order='C')
if self.ds.img_mini_T[b_idx][mini_idx].shape[1] < bsz:
tmp = self.ds.img_mini_T[b_idx][mini_idx].shape[0]
mb_residual = self.ds.img_mini_T[b_idx][mini_idx].shape[1]
filledbatch = np.vstack((img_macro[s_idx:e_idx],
np.zeros((bsz - mb_residual, tmp))))
self.ds.img_mini_T[b_idx][mini_idx] = \
filledbatch.T.astype(betype, order='C')
for lbl in self.ds.label_list:
hl = np.squeeze(lbl_macro[lbl][s_idx:e_idx])
self.ds.lbl_one_hot[b_idx][lbl][mini_idx] = \
np.eye(self.ds.nclass[lbl])[hl].T.astype(betype, order='C')
return
def run(self):
import imgworker
bsz = self.ds.batch_size
batch_idx = self.ds.macro_decode_buf_idx
jdict = self.ds.get_macro_batch()
ibetype = self.ds.img_dtype
# This macrobatch could be smaller than macro_size for last macrobatch
macro_size = len(jdict['data'])
self.macro_size = macro_size
if self.ds.shuffle_macro:
shuff_idx = range(macro_size)
shuffle(shuff_idx)
jdict['data'] = [jdict['data'][i] for i in shuff_idx]
for k in self.ds.label_list:
jdict['labels'][k] = [jdict['labels'][k][i] for i in shuff_idx]
self.ds.target_macro[batch_idx] = \
jdict['targets'] if 'targets' in jdict else None
lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list}
do_center = self.ds.predict or not self.ds.dotransforms
do_flip = self.ds.dotransforms
imgworker.decode_list(jpglist=jdict['data'],
tgt=self.ds.int_macro[batch_idx][:macro_size],
orig_size=self.ds.output_image_size,
crop_size=self.ds.cropped_image_size,
center=do_center, flip=do_flip,
rgb=self.ds.rgb,
nthreads=self.ds.num_workers)
# This ensures we take the last partial minibatch
self.ds.minis_per_macro[batch_idx] = -(-macro_size // bsz)
for lbl in self.ds.label_list:
hl = np.squeeze(lbl_macro[lbl])
self.ds.lbl_one_hot[batch_idx][lbl][:macro_size] = \
np.eye(self.ds.nclass[lbl],
dtype=ibetype)[hl].astype(ibetype, order='C')
if macro_size < self.ds.macro_size:
self.ds.lbl_one_hot[batch_idx][lbl][macro_size:] = 0
np.subtract(self.ds.int_macro[batch_idx], self.ds.mean_val,
self.ds.img_macro[batch_idx])
if macro_size < self.ds.macro_size:
self.ds.img_macro[batch_idx][macro_size:] = 0
return
def run(self):
import imgworker
bsz = self.ds.batch_size
batch_idx = self.ds.macro_decode_buf_idx
jdict = self.ds.get_macro_batch()
ibetype = self.ds.img_dtype
# This macrobatch could be smaller than macro_size for last macrobatch
macro_size = len(jdict['data'])
self.macro_size = macro_size
if self.ds.shuffle_macro:
shuff_idx = range(macro_size)
shuffle(shuff_idx)
jdict['data'] = [jdict['data'][i] for i in shuff_idx]
for k in self.ds.label_list:
jdict['labels'][k] = [jdict['labels'][k][i] for i in shuff_idx]
self.ds.target_macro[batch_idx] = \
jdict['targets'] if 'targets' in jdict else None
lbl_macro = {k: jdict['labels'][k] for k in self.ds.label_list}
do_center = self.ds.predict or not self.ds.dotransforms
do_flip = self.ds.dotransforms
imgworker.decode_list(jpglist=jdict['data'],
tgt=self.ds.int_macro[batch_idx][:macro_size],
orig_size=self.ds.output_image_size,
crop_size=self.ds.cropped_image_size,
center=do_center, flip=do_flip,
rgb=self.ds.rgb,
nthreads=self.ds.num_workers)
# This ensures we take the last partial minibatch
self.ds.minis_per_macro[batch_idx] = -(-macro_size // bsz)
for lbl in self.ds.label_list:
hl = np.squeeze(lbl_macro[lbl])
self.ds.lbl_one_hot[batch_idx][lbl][:macro_size] = \
np.eye(self.ds.nclass[lbl],
dtype=ibetype)[hl].astype(ibetype, order='C')
if macro_size < self.ds.macro_size:
self.ds.lbl_one_hot[batch_idx][lbl][macro_size:] = 0
np.subtract(self.ds.int_macro[batch_idx], self.ds.mean_val,
self.ds.img_macro[batch_idx])
if macro_size < self.ds.macro_size:
self.ds.img_macro[batch_idx][macro_size:] = 0
return
# a[:] = labels
numthreads = int(sys.argv[1])
imgsize = 256
innersize = 224
innerpixels = innersize * innersize * 3
startimg = 0
num_imgs = 2048
unpacked = np.zeros((num_imgs, innerpixels), dtype=np.uint8)
t0 = time.time()
for i in range(24):
t1 = time.time()
PATH = '/usr/local/data/I1K/macro_batches_256/training_batch_{:d}/'.format(
i)
fname = os.path.join(PATH, 'training_batch_{:d}.0'.format(i))
tdata = my_unpickle(fname)
jpeglist = tdata['data']
# labels = tdata['labels']
iw.decode_list(jpglist=jpeglist[startimg:startimg + num_imgs],
tgt=unpacked,
orig_size=imgsize,
crop_size=innersize,
nthreads=numthreads)
print("\tC Module version decode time: ", time.time() - t1)
print("Total Time:", time.time() - t0)
# Now save out an image from the packed array to confirm things are working
data = unpacked[5, :].reshape(
(3, innersize, innersize)).transpose(1, 2, 0)[:, :, [0, 1, 2]]
im = Image.fromarray(data)
im.save('out.png')
# labels = [item for sublist in labels for item in sublist]
# labels = np.asarray(labels, dtype='float32')
# a = np.empty((1,3072), dtype=np.float32)
# a[:] = labels
numthreads = int(sys.argv[1])
imgsize = 256
innersize = 224
innerpixels = innersize*innersize*3
startimg = 0
num_imgs = 2048
unpacked = np.zeros((num_imgs, innerpixels), dtype=np.uint8)
t0 = time.time()
for i in range(24):
t1 = time.time()
PATH = '/usr/local/data/I1K/macro_batches_256/training_batch_{:d}/'.format(i)
fname = os.path.join(PATH, 'training_batch_{:d}.0'.format(i))
tdata = my_unpickle(fname)
jpeglist = tdata['data']
# labels = tdata['labels']
iw.decode_list(jpglist=jpeglist[startimg:startimg+num_imgs],
tgt=unpacked, orig_size=imgsize,
crop_size=innersize, nthreads=numthreads)
print("\tC Module version decode time: ", time.time() - t1)
print("Total Time:", time.time() - t0)
# Now save out an image from the packed array to confirm things are working
data = unpacked[5,:].reshape(
(3, innersize, innersize)).transpose(1,2,0)[:,:,[0,1,2]]
im = Image.fromarray(data)
im.save('out.png')
