def get_valloader_only(toy,
rsyncing,
batch_size=16,
num_workers=os.cpu_count() - 1,
notebook=False,
cat=False):
"""
:param toy:
:param rsyncing:
:param batch_size:
:param num_workers:
:param notebook:
:return:
"""
num_workers = 0
if rsyncing:
print('Rsynced data! (prepare feat)', flush=True)
else:
print('Using symbolic links! (prepare feat)', flush=True)
print('Getting path ready..', flush=True)
_, anno_path_val, png_path = get_paths(rsyncing, toy, notebook)
start_time = time.time()
print('Creating Coco Dataset..', flush=True)
if not cat:
valset = u.dataset_coco(
png_path,
anno_path_val,
transform=torchvision.transforms.Compose(
[t.Normalize(), t.BboxCrop(targetsize=224)]),
bbox_transform=torchvision.transforms.Compose([t.GetFiveBBs()]),
for_feature=True,
cat=cat)
else:
valset = u.dataset_coco(
png_path,
anno_path_val,
transform=torchvision.transforms.Compose(
[t.Normalize(), t.BboxCropMult(targetsize=224)]),
bbox_transform=torchvision.transforms.Compose([t.GetBBsMult()]),
for_feature=True,
cat=cat)
print('Validation set has', len(valset), 'images', flush=True)
valloader = torch.utils.data.DataLoader(valset,
batch_size=batch_size,
sampler=SequentialSampler(valset),
num_workers=num_workers,
collate_fn=u.mammo_collate)
print('Validation loader has', len(valloader), 'batches', flush=True)
total_time = time.time() - start_time
print('Creating Datasets took {:.0f} seconds.'.format(total_time),
flush=True)
return valloader
def get_wsl_cam(img_name, model='deepmil_multi'):
wsl_dir = '/home/victor/PycharmProjects/active-learning-segmentation-baselines/wsl_cams'
cams_dir = join(wsl_dir, '{}/npy/'.format(model))
paths = get_paths(cams_dir, 'npy')
file_names = get_files(cams_dir, 'npy')
file_names = [f.replace('.npy', '') for f in file_names]
wsl_cam_path = paths[file_names.index(img_name[0].replace('.bmp', ''))]
return np.load(wsl_cam_path)
def get_envs(cfg_dict, model, rsyncing, toy=False,f_one=False):
if cfg_dict.EXPERIMENTAL_ENV:
from environment_exp import MammoEnv
else:
from environment import MammoEnv
one_img, max_num_imgs_train, max_num_imgs_val, _, _, _, _, _, dist_reward, dist_factor, with_hist = get_q_variants_oneImg_maxNumImgsT_maxNumImgsV_double_combi_paramNoise_recurrent_recSize_distReward_distFactor_hist(
cfg_dict)
_, zeta, eta, iota, _ = get_q_rewards_gamma_zeta_eta_iota_clipVal(cfg_dict)
tau = get_q_explo_kappa_epochsEps_targetEps_tau_testTau_tauEpochs(cfg_dict)[3]
print('Getting path ready..', flush=True)
anno_path_train, anno_path_val, png_path = get_paths(rsyncing, toy)
print('Creating Coco Datasets..', flush=True)
if one_img:
# png_path = os.path.join('/mnt/synology/breast/projects/lisa/koel/one_img_dataset_small', 'png')
# anno_path_train = os.path.join('/mnt/synology/breast/projects/lisa/koel/one_img_dataset_small',
# 'annotations/mscoco_train_full.json')
# png_path = os.path.join('/mnt/synology/breast/projects/lisa/koel/one_img_dataset', 'png')
# anno_path_train = os.path.join('/mnt/synology/breast/projects/lisa/koel/one_img_dataset',
# 'annotations/mscoco_train_full.json')
png_path = os.path.join('../one_img_dataset', 'png')
anno_path_train = os.path.join('../one_img_dataset',
'annotations/mscoco_train_full.json')
# TODO
# png_path = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset', 'png')
# anno_path_train = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
trainset = u.dataset_coco(png_path, anno_path_train, add_border=get_addBorder(cfg))
train_env = MammoEnv(trainset, eta=eta, zeta=zeta, tau=tau, model=model, one_img=one_img,
max_no_imgs=max_num_imgs_train, iota=iota,dist_reward = dist_reward, dist_factor=dist_factor, with_hist=with_hist)
val_env = MammoEnv(trainset, eta=eta, zeta=zeta, tau=tau, model=model, one_img=one_img,
max_no_imgs=max_num_imgs_val, iota=iota,dist_reward = dist_reward, dist_factor=dist_factor, with_hist=with_hist)
else:
# TODO transform for downsampling
print('Building training set', flush=True)
trainset = u.dataset_coco(png_path, anno_path_train, add_border=get_addBorder(cfg))
print('Training set has', len(trainset), 'images', flush=True)
print('Building validation set', flush=True)
valset = u.dataset_coco(png_path, anno_path_val, add_border=get_addBorder(cfg),f_one=f_one)
print('Validation set has', len(valset), 'images', flush=True)
print('Make train env', flush=True)
train_env = MammoEnv(trainset, eta=eta, zeta=zeta, tau=tau, model=model, one_img=one_img,
max_no_imgs=max_num_imgs_train, iota=iota,dist_reward = dist_reward, dist_factor=dist_factor,with_hist=with_hist)
print('Make val env', flush=True)
val_env = MammoEnv(valset, eta=eta, zeta=zeta, tau=tau, model=model, one_img=one_img,
max_no_imgs=max_num_imgs_val, iota=iota,dist_reward = dist_reward, dist_factor=dist_factor,with_hist=with_hist,f_one=f_one)
return train_env, val_env
def test_screenpoint3():
checkpoint_dir = '../../checkpoints_from_18_07_18/qnet_23'
print(os.path.abspath(checkpoint_dir))
rsyncing = False
batch_size = 1
multiprocessing = False
# num_workers=os.cpu_count() - 1
num_workers = 0
data_aug_vec = [0.5, 0.25, 0.5, 0.5]
toy = False
notebook = False
anno_path_train, anno_path_val, png_path = get_paths(
rsyncing, toy, notebook)
# TODO
# png_path = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset', 'png')
# anno_path_train = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
# anno_path_val = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
print('Creating Coco Datasets..', flush=True)
# t.ToTensor()
trans_img = torchvision.transforms.Compose([
t.BboxCrop(targetsize=224),
t.RandomFlipImg(prob=data_aug_vec[0]),
t.RandomGammaImg(prob=data_aug_vec[1], use_normal_distribution=True)
])
trans_bb = torchvision.transforms.Compose([
t.GetFourBBs(),
t.RandomTranslateBB(prob=data_aug_vec[2], pixel_range=10),
t.RandomScaleBB(prob=data_aug_vec[3], max_percentage=0.1)
])
trainset = u.dataset_coco(png_path,
anno_path_train,
transform=trans_img,
bbox_transform=trans_bb)
print('Training set has', len(trainset), 'images', flush=True)
valset = u.dataset_coco(png_path,
anno_path_val,
transform=torchvision.transforms.Compose(
[t.GetFourBBs(),
t.BboxCrop(targetsize=224)]))
print('Validation set has', len(valset), 'images', flush=True)
print('Len trainset', len(trainset))
print('Len valset', len(valset))
print('Example set', trainset[3])
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=SequentialSampler(trainset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
valloader = torch.utils.data.DataLoader(valset,
batch_size=batch_size,
sampler=SequentialSampler(valset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
print('Len trainloader', len(trainloader))
print('Len valloader', len(valloader))
empty_batches = 0
filled_batches = 0
for idx, batch in enumerate(trainloader):
# print('----')
# print('{} th batch of {}'.format(idx,len(trainloader)),flush=True)
# if len(batch['image_batch']) == 0:
# empty_batches += 1
# else:
# filled_batches += 1
# print(batch)
# break
pass
def change_width_height_bbox():
print('Get paths')
anno_path_train, anno_path_val, _ = get_paths(False, toy=True)
print('Open json')
def get_sequential_trainloader(toy,
rsyncing,
batch_size=16,
num_workers=os.cpu_count() - 1,
data_aug_vec=[0.5, 0.25, 0.5, 0.5],
notebook=False):
"""
:param toy:
:param rsyncing:
:param batch_size:
:param num_workers:
:param data_aug_vec:
:param notebook:
:return:
"""
num_workers = 0
if rsyncing:
print('Rsynced data! (prepare feat)', flush=True)
else:
print('Using symbolic links! (prepare feat)', flush=True)
print('Getting path ready..', flush=True)
anno_path_train, _, png_path = get_paths(rsyncing, toy, notebook)
# TODO
# png_path = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset', 'png')
# anno_path_train = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
trans_img = torchvision.transforms.Compose([
t.Normalize(),
t.BboxCrop(targetsize=224),
t.RandomFlipImg(prob=data_aug_vec[0]),
t.RandomGammaImg(prob=data_aug_vec[1], use_normal_distribution=True)
])
trans_bb = torchvision.transforms.Compose([
t.GetFiveBBs(),
t.RandomTranslateBB(prob=data_aug_vec[2], pixel_range=10),
t.RandomScaleBB(prob=data_aug_vec[3], max_percentage=0.1)
])
start_time = time.time()
print('Creating Coco Dataset..', flush=True)
trainset = u.dataset_coco(png_path,
anno_path_train,
transform=trans_img,
bbox_transform=trans_bb,
for_feature=True)
print('Training set has', len(trainset), 'images', flush=True)
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=SequentialSampler(trainset),
num_workers=num_workers,
collate_fn=u.mammo_collate)
print('Training loader has', len(trainloader), 'batches', flush=True)
total_time = time.time() - start_time
print('Creating Datasets took {:.0f} seconds.'.format(total_time),
flush=True)
return trainloader
def get_dataloaders(checkpoint_dir,
rsyncing,
selective_sampling=False,
warmup_trainer=None,
batch_size=16,
num_workers=os.cpu_count() - 1,
data_aug_vec=[0.5, 0.25, 0.5, 0.5],
toy=False,
notebook=False,
cat=False):
"""
:param checkpoint_dir:
:param rsyncing:
:param selective_sampling:
:param warmup_trainer:
:param batch_size:
:param num_workers:
:param seed:
:param data_aug_vec: probabilities for rnd flip, rnd gamma, rnd translation and rnd scale
:param toy:
:param notebook:
:return:
"""
# if torch.cuda.is_available():
# mp.set_start_method('spawn')
multiprocessing = False
num_workers = 0
sampler_size = 3000
if rsyncing:
print('Rsynced data! (prepare feat)', flush=True)
else:
print('Using symbolic links! (prepare feat)', flush=True)
print('Getting path ready..', flush=True)
anno_path_train, anno_path_val, png_path = get_paths(
rsyncing, toy, notebook)
# TODO
# png_path = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset', 'png')
# anno_path_train = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
# anno_path_val = os.path.join('/Users/lisa/Documents/Uni/ThesisDS/thesis_ds/one_img_dataset',
# 'annotations/mscoco_train_full.json')
print('Creating Coco Datasets..', flush=True)
# t.ToTensor()
if not cat:
trans_img = torchvision.transforms.Compose([
t.Normalize(),
t.BboxCrop(targetsize=224),
t.RandomFlipImg(prob=data_aug_vec[0]),
t.RandomGammaImg(prob=data_aug_vec[1],
use_normal_distribution=True)
])
trans_bb = torchvision.transforms.Compose([
t.GetFiveBBs(),
t.RandomTranslateBB(prob=data_aug_vec[2], pixel_range=10),
t.RandomScaleBB(prob=data_aug_vec[3], max_percentage=0.1)
])
else:
trans_img = torchvision.transforms.Compose([
t.Normalize(),
t.BboxCropMult(targetsize=224),
t.RandomFlipImg(prob=data_aug_vec[0]),
t.RandomGammaImg(prob=data_aug_vec[1],
use_normal_distribution=True)
])
trans_bb = torchvision.transforms.Compose([
t.GetBBsMult(),
t.RandomTranslateBB(prob=data_aug_vec[2], pixel_range=10,
cat=True),
t.RandomScaleBB(prob=data_aug_vec[3], max_percentage=0.1, cat=True)
])
trainset = u.dataset_coco(png_path,
anno_path_train,
transform=trans_img,
bbox_transform=trans_bb,
for_feature=True,
cat=cat)
print('Training set has', len(trainset), 'images', flush=True)
if not cat:
valset = u.dataset_coco(
png_path,
anno_path_val,
transform=torchvision.transforms.Compose(
[t.Normalize(), t.BboxCrop(targetsize=224)]),
bbox_transform=torchvision.transforms.Compose([t.GetFiveBBs()]),
for_feature=True,
cat=cat)
else:
valset = u.dataset_coco(
png_path,
anno_path_val,
transform=torchvision.transforms.Compose(
[t.Normalize(), t.BboxCropMult(targetsize=224)]),
bbox_transform=torchvision.transforms.Compose([t.GetBBsMult()]),
for_feature=True,
cat=cat)
print('Validation set has', len(valset), 'images', flush=True)
if selective_sampling:
if not warmup_trainer:
print(
'Cannot calculate weights for selective sampling: no model given. Using normal sampling instead',
flush=True)
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=RandomSampler(trainset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
else:
print('Getting weights for sampling..', flush=True)
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=SequentialSampler(trainset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
weights = warmup_trainer.predict_dataset(trainloader)
pkl.dump(
weights,
open(
os.path.join(checkpoint_dir,
'weights_selective_train.pkl'), 'wb'))
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=WeightedRandomSampler(weights.double(),
sampler_size,
replacement=False),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
else:
trainloader = torch.utils.data.DataLoader(
trainset,
batch_size=batch_size,
sampler=RandomSampler(trainset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
valloader = torch.utils.data.DataLoader(valset,
batch_size=batch_size,
sampler=SequentialSampler(valset),
num_workers=num_workers,
collate_fn=u.mammo_collate,
pin_memory=multiprocessing)
print('Training loader has', len(trainloader), 'batches', flush=True)
print('Validation loader has', len(valloader), 'batches', flush=True)
return trainloader, valloader