def calc_statistics(loader='train'):
"""
calculate mean and std for data sets
please close normalize in dataset's transform manually
:return:
"""
from dl_backbone.data.build import make_data_loader
if loader == 'train':
data_loader = make_data_loader(cfg, cfg.DATASETS.TRAIN, is_train=True)
elif loader == 'valid':
data_loader = make_data_loader(cfg, cfg.DATASETS.VALID, is_train=False)
elif loader == 'test':
data_loader = make_data_loader(cfg, cfg.DATASETS.TEST, is_train=False)
else:
raise KeyError('loader must be specified')
mean = 0.
std = 0.
nb_samples = 0.
for iteration, (images, targets, indices) in enumerate(data_loader):
batch_size = images.size(0)
images = images.view(batch_size, images.size(1), -1)
mean += images.mean(2).sum(0)
std += images.std(2).sum(0)
nb_samples += batch_size
running_mean = mean / nb_samples
running_std = std / nb_samples
print("iter %d running mean: " % iteration, running_mean)
print("iter %d running std : " % iteration, running_std)
def test_train_loader(cfg):
cfg.SOLVER.IMS_PER_BATCH = 10
from dl_backbone.data.build import make_data_loader
data_loader = make_data_loader(cfg, cfg.DATASETS.TRAIN, is_train=False)
for iteration, (images, targets, indices) in enumerate(data_loader):
print("Image Batch Size: ", images.size())
print("Target Batch Size: ", targets.size())
print("Indices: ", indices)
for idx in range(len(images)):
np_img = images[idx][[2, 1, 0], :, :].permute(1, 2,
0).numpy() * 255
cv2.imwrite('train_output_%d.jpg' % idx, np_img)
break
def test_weighted_sampler(cfg):
cfg.SOLVER.IMS_PER_BATCH = 64
from dl_backbone.data.build import make_data_loader
data_loader = make_data_loader(cfg, cfg.DATASETS.TRAIN, is_train=True)
count = np.zeros(cfg.MODEL.NUM_CLASS)
total_label = 0
for iteration, (images, targets, indices) in enumerate(data_loader):
for idx in range(len(targets)):
target_vec = targets[idx]
labels = np.squeeze(torch.nonzero(target_vec).numpy()).tolist()
if isinstance(labels, int):
labels = [labels]
total_label += len(labels)
for label in labels:
count[label] += 1
np.set_printoptions(precision=1)
ratio = (count / total_label) * 100
print(ratio)