def check_preprocessed_data(data_access, dataset, targets, batch_size, tmp_size, final_size, preprocessing_func,
preprocessing_args, n=10):
if data_access=="in-memory":
train_dataset = InMemoryDataset("train", source=dataset, batch_size=batch_size, source_targets=targets)
elif data_access=="fuel":
train_dataset = FuelDataset("test", tmp_size, batch_size=batch_size, division="leaderboard", shuffle=False)
else:
raise Exception("Data access not available. Must be 'fuel' or 'in-memory'. Here : %s."%data_access)
# Compute only one batch
start=time.time()
batch,batch_targets = train_dataset.get_batch()
batch_targets = convert_labels(batch_targets)
processed_batch = np.zeros((batch.shape[0],final_size[2],final_size[0],final_size[1]),
dtype="float32")
for k in range(batch_size):
processed_batch[k] = preprocessing_func(batch[k], *preprocessing_args).transpose(2,0,1)
end=time.time()
print "Batch Shape = ", processed_batch.shape, "with dtype =", processed_batch.dtype
print "Targets Shape =", batch_targets.shape, "with dtype =", batch_targets.dtype
for i in range(n):
plt.figure(0)
plt.gray()
plt.clf()
plt.title("(%d,%d)"%(batch_targets[i][0], batch_targets[i][1]))
if batch.shape[1]==3:
plt.imshow(processed_batch[i].transpose(1,2,0))
else:
plt.imshow(processed_batch[i,0])
plt.show()
print "Processing 1 batch took : %.5f"%(end-start)
def check_preprocessed_data(data_access,
dataset,
targets,
batch_size,
tmp_size,
final_size,
preprocessing_func,
preprocessing_args,
n=10):
if data_access == "in-memory":
train_dataset = InMemoryDataset("train",
source=dataset,
batch_size=batch_size,
source_targets=targets)
elif data_access == "fuel":
train_dataset = FuelDataset("test",
tmp_size,
batch_size=batch_size,
division="leaderboard",
shuffle=False)
else:
raise Exception(
"Data access not available. Must be 'fuel' or 'in-memory'. Here : %s."
% data_access)
# Compute only one batch
start = time.time()
batch, batch_targets = train_dataset.get_batch()
batch_targets = convert_labels(batch_targets)
processed_batch = np.zeros(
(batch.shape[0], final_size[2], final_size[0], final_size[1]),
dtype="float32")
for k in range(batch_size):
processed_batch[k] = preprocessing_func(batch[k],
*preprocessing_args).transpose(
2, 0, 1)
end = time.time()
print "Batch Shape = ", processed_batch.shape, "with dtype =", processed_batch.dtype
print "Targets Shape =", batch_targets.shape, "with dtype =", batch_targets.dtype
for i in range(n):
plt.figure(0)
plt.gray()
plt.clf()
plt.title("(%d,%d)" % (batch_targets[i][0], batch_targets[i][1]))
if batch.shape[1] == 3:
plt.imshow(processed_batch[i].transpose(1, 2, 0))
else:
plt.imshow(processed_batch[i, 0])
plt.show()
print "Processing 1 batch took : %.5f" % (end - start)
