def test_data_loader(cfg, fname):
dims = cfg['dims']
chunk_size = cfg['batch_size'] * cfg['batches_per_chunk']
xc = np.zeros((
chunk_size,
cfg['n_channels'],
) + dims, dtype=np.float32)
reader = npytar.NpyTarReader(fname)
yc = np.zeros((chunk_size, cfg['n_classes']), dtype=np.float32)
counter = []
for ix, (x, name) in enumerate(reader):
cix = ix % chunk_size
xc[cix] = x.astype(np.float32)
yc[cix, (int(name.split('.')[0]) - 1)] = 1
counter.append(int(name.split('.')[0]) - 1)
if len(counter) == chunk_size:
yield (3.0 * xc - 1.0, yc)
counter = []
yc.fill(0)
xc.fill(0)
if len(counter) > 0:
# pad to nearest multiple of batch_size
if len(counter) % cfg['batch_size'] != 0:
new_size = int(np.ceil(
len(counter) / float(cfg['batch_size']))) * cfg['batch_size']
xc = xc[:new_size]
xc[len(counter):] = xc[:(new_size - len(counter))]
yc = yc[:new_size]
yc[len(counter):] = yc[:(new_size - len(counter))]
counter = counter + counter[:(new_size - len(counter))]
yield (3.0 * xc - 1.0, yc)
def data_loader(fname):
reader = npytar.NpyTarReader(fname)
xc = np.zeros((reader.length(), ) + input_shape, dtype=np.float32)
reader.reopen()
for ix, (x, name) in enumerate(reader):
xc[ix] = x.astype(np.float32)
return 3.0 * xc - 1.0
def __init__(self, filedir, filename):
reader = npytar.NpyTarReader(os.path.join(filedir, filename))
temp_list = []
for (arr, name) in reader:
temp_list.append((arr, name))
reader.close()
# in this dataset we have 12 views for each shape
if len(temp_list) % 12 != 0:
# assert is a statement in python2/3, not a function
assert "some shapes might not have 12 views"
# b x v x c x d x d x d
self.data = np.zeros((len(temp_list) // 12, 12, 1, 32, 32, 32),
dtype=np.float32)
# all view share the same label
self.label = np.zeros((len(temp_list) // 12, ), dtype=np.int)
# sort the file by its name
# format: classnum.classname_idx.viewidx
# exception: 001.2h31k8fak.viewidx
temp_list.sort(key=lambda x: (int(x[1].split(".")[0]), x[1].split(".")[
-2].split("_")[-1], int(x[1].split(".")[-1])))
for idx, (arr, name) in enumerate(temp_list):
self.data[idx // 12, idx % 12, 0] = arr
if idx % 12 == 0:
# assign label
# name: class_idx.fname.view_idx
self.label[idx // 12] = int(name.split('.')[0]) - 1
else:
# check label consistency
assert self.label[idx // 12] == (
int(name.split('.')[0]) - 1
), "label is inconsistent among different views for file {}, original label{}".format(
name, self.label[idx // 12])
def data_loader(fname):
x_dic = {}
reader = npytar.NpyTarReader(fname)
for ix, (x, name) in enumerate(reader):
x_dic[name] = x.astype(np.float32)
reader.reopen()
xc = np.zeros((reader.length(), ) + input_shape, dtype=np.float32)
i = 0
for ik in sorted(x_dic.keys(), key=natural_keys):
xc[i] = x_dic[ik]
i += 1
return xc
def data_loader(cfg, fname):
dims = cfg['dims']
chunk_size = cfg['batch_size'] * cfg['batches_per_chunk'] // 2
xc = np.zeros((
chunk_size,
cfg['n_channels'],
) + dims, dtype=np.float32)
reader = npytar.NpyTarReader(fname)
yc = np.zeros((chunk_size, cfg['n_classes']), dtype=np.float32)
counter = []
for ix, (x, name) in enumerate(reader):
cix = ix % chunk_size
xc[cix] = x.astype(np.float32)
yc[cix, (int(name.split('.')[0]) - 1)] = 1
counter.append(int(name.split('.')[0]) - 1)
if len(counter) == chunk_size:
indices = np.random.permutation(2 * len(xc))
yield (
3.0 * np.append(xc, jitter_chunk(xc, cfg), axis=0)[indices] -
1.0, np.append(yc, yc, axis=0)[indices])
counter = []
yc.fill(0)
xc.fill(0)
if len(counter) > 0:
# pad to nearest multiple of batch_size
if len(counter) % cfg['batch_size'] != 0:
new_size = int(np.ceil(
len(counter) / float(cfg['batch_size']))) * cfg['batch_size']
xc = xc[:new_size]
xc[len(counter):] = xc[:(new_size - len(counter))]
yc = yc[:new_size]
yc[len(counter):] = yc[:(new_size - len(counter))]
counter = counter + counter[:(new_size - len(counter))]
indices = np.random.permutation(2 * len(xc))
yield (3.0 * np.append(xc, jitter_chunk(xc, cfg), axis=0)[indices] -
1.0, np.append(yc, yc, axis=0)[indices])