def manual_check(self):
""" print examples from a toy batch to file.
:return:
"""
self.ra_ext = utils.import_module("ra_ext", 'custom_extensions/roi_align/roi_align.py')
# actual mrcnn mask input
from datasets.toy import configs
cf = configs.Configs()
cf.exp_dir = "datasets/toy/experiments/dev/"
cf.plot_dir = cf.exp_dir + "plots"
os.makedirs(cf.exp_dir, exist_ok=True)
cf.fold = 0
cf.n_workers = 1
logger = utils.get_logger(cf.exp_dir)
data_loader = utils.import_module('data_loader', os.path.join("datasets", "toy", 'data_loader.py'))
batch_gen = data_loader.get_train_generators(cf, logger=logger)
batch = next(batch_gen['train'])
roi_mask = np.zeros((1, 320, 200))
bb_target = (np.array([50, 40, 90, 120])).astype("int")
roi_mask[:, bb_target[0]+1:bb_target[2]+1, bb_target[1]+1:bb_target[3]+1] = 1.
#batch = {"roi_masks": np.array([np.array([roi_mask, roi_mask]), np.array([roi_mask])]), "bb_target": [[bb_target, bb_target + 25], [bb_target-20]]}
#batch_boxes_cor = [torch.tensor(batch_el_boxes).cuda().float() for batch_el_boxes in batch_cor["bb_target"]]
batch_boxes = [torch.tensor(batch_el_boxes).cuda().float() for batch_el_boxes in batch["bb_target"]]
#import IPython; IPython.embed()
for b in range(len(batch_boxes)):
roi_masks = batch["roi_masks"][b]
#roi_masks_cor = batch_cor["roi_masks"][b]
if roi_masks.sum()>0:
boxes = batch_boxes[b]
roi_masks = torch.tensor(roi_masks).cuda().type(dtype=torch.float32)
box_ids = torch.arange(roi_masks.shape[0]).cuda().unsqueeze(1).type(dtype=torch.float32)
masks = tv.ops.roi_align(roi_masks, [boxes], cf.mask_shape)
masks = masks.squeeze(1)
masks = torch.round(masks)
masks_own = self.ra_ext.roi_align_2d(roi_masks, torch.cat((box_ids, boxes), dim=1), cf.mask_shape)
boxes = boxes.type(torch.int)
#print("check roi mask", roi_masks[0, 0, boxes[0][0]:boxes[0][2], boxes[0][1]:boxes[0][3]].sum(), (boxes[0][2]-boxes[0][0]) * (boxes[0][3]-boxes[0][1]))
#print("batch masks", batch["roi_masks"])
masks_own = masks_own.squeeze(1)
masks_own = torch.round(masks_own)
#import IPython; IPython.embed()
for mix, mask in enumerate(masks):
fig = plg.plt.figure()
ax = fig.add_subplot()
ax.imshow(roi_masks[mix][0].cpu().numpy(), cmap="gray", vmin=0.)
ax.axis("off")
y1, x1, y2, x2 = boxes[mix]
bbox = plg.mpatches.Rectangle((x1, y1), x2-x1, y2-y1, linewidth=0.9, edgecolor="c", facecolor='none')
ax.add_patch(bbox)
x1, y1, x2, y2 = boxes[mix]
bbox = plg.mpatches.Rectangle((x1, y1), x2 - x1, y2 - y1, linewidth=0.9, edgecolor="r",
facecolor='none')
ax.add_patch(bbox)
debug_dir = Path("/home/gregor/Documents/regrcnn/datasets/toy/experiments/debugroial")
os.makedirs(debug_dir, exist_ok=True)
plg.plt.savefig(debug_dir/"mask_b{}_{}.png".format(b, mix))
plg.plt.imsave(debug_dir/"mask_b{}_{}_pooled_tv.png".format(b, mix), mask.cpu().numpy(), cmap="gray", vmin=0.)
plg.plt.imsave(debug_dir/"mask_b{}_{}_pooled_own.png".format(b, mix), masks_own[mix].cpu().numpy(), cmap="gray", vmin=0.)
return
def build(self):
"""Build Retina Net architecture."""
# Image size must be dividable by 2 multiple times.
h, w = self.cf.patch_size[:2]
if h / 2 ** 5 != int(h / 2 ** 5) or w / 2 ** 5 != int(w / 2 ** 5):
raise Exception("Image size must be divisible by 2 at least 5 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
backbone = utils.import_module('bbone', self.cf.backbone_path)
self.logger.info("loaded backbone from {}".format(self.cf.backbone_path))
conv = backbone.ConvGenerator(self.cf.dim)
# build Anchors, FPN, Classifier / Bbox-Regressor -head
self.np_anchors = mutils.generate_pyramid_anchors(self.logger, self.cf)
self.anchors = torch.from_numpy(self.np_anchors).float().cuda()
self.fpn = backbone.FPN(self.cf, conv, operate_stride1=self.cf.operate_stride1).cuda()
self.classifier = Classifier(self.cf, conv).cuda()
self.bb_regressor = BBRegressor(self.cf, conv).cuda()
if 'regression' in self.cf.prediction_tasks:
self.roi_regressor = RoIRegressor(self.cf, conv, self.cf.regression_n_features).cuda()
elif 'regression_bin' in self.cf.prediction_tasks:
# classify into bins of regression values
self.roi_regressor = RoIRegressor(self.cf, conv, len(self.cf.bin_labels)).cuda()
else:
self.roi_regressor = lambda x: [torch.tensor([]).cuda()]
if self.cf.model == 'retina_unet':
self.final_conv = conv(self.cf.end_filts, self.cf.num_seg_classes, ks=1, pad=0, norm=None, relu=None)
def __init__(self, cf, logger):
super(net, self).__init__()
self.cf = cf
self.logger = logger
backbone = utils.import_module('bbone', cf.backbone_path)
self.logger.info("loaded backbone from {}".format(
self.cf.backbone_path))
conv_gen = backbone.ConvGenerator(cf.dim)
# set operate_stride1=True to generate a unet-like FPN.)
self.fpn = backbone.FPN(cf,
conv=conv_gen,
relu_enc=cf.relu,
operate_stride1=True)
self.conv_final = conv_gen(cf.end_filts,
cf.num_seg_classes,
ks=1,
pad=0,
norm=cf.norm,
relu=None)
#initialize parameters
if self.cf.weight_init == "custom":
logger.info(
"Tried to use custom weight init which is not defined. Using pytorch default."
)
elif self.cf.weight_init:
mutils.initialize_weights(self)
else:
logger.info("using default pytorch weight init")
def test(self, n_cases=200, box_count=30, threshold=0.5):
# dynamically import module so that it doesn't affect other tests if import fails
self.nms_ext = utils.import_module("nms_ext",
'custom_extensions/nms/nms.py')
self.manual_example()
# change seed to something fix if you want exactly reproducible test
seed0 = np.random.randint(50)
print("NMS test progress (done/total box configurations) 2D:",
end="\n")
for i in tqdm.tqdm(range(n_cases)):
self.single_case(count=box_count,
dim=2,
threshold=threshold,
seed=seed0 + i)
print("NMS test progress (done/total box configurations) 3D:",
end="\n")
for i in tqdm.tqdm(range(n_cases)):
self.single_case(count=box_count,
dim=3,
threshold=threshold,
seed=seed0 + i)
return
def build(self):
"""
Build Retina Net architecture.
"""
# Image size must be dividable by 2 multiple times.
h, w = self.cf.patch_size[:2]
if h / 2**5 != int(h / 2**5) or w / 2**5 != int(w / 2**5):
raise Exception(
"Image size must be dividable by 2 at least 5 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# instanciate abstract multi dimensional conv class and backbone model.
conv = mutils.NDConvGenerator(self.cf.dim)
backbone = utils.import_module('bbone', self.cf.backbone_path)
# build Anchors, FPN, Classifier / Bbox-Regressor -head
self.np_anchors = mutils.generate_pyramid_anchors(self.logger, self.cf)
self.anchors = torch.from_numpy(self.np_anchors).float().cuda()
self.Fpn = backbone.FPN(self.cf,
conv,
operate_stride1=self.cf.operate_stride1)
self.Classifier = Classifier(self.cf, conv)
self.BBRegressor = BBRegressor(self.cf, conv)
self.final_conv = conv(self.cf.end_filts,
self.cf.num_seg_classes,
ks=1,
pad=0,
norm=self.cf.norm,
relu=None)
def specific_example_check(self):
# dummy input
self.ra_ext = utils.import_module("ra_ext", 'custom_extensions/roi_align/roi_align.py')
exp = 6
pool_size = (2,2)
fmap = torch.arange(exp**2).view(exp,exp).unsqueeze(0).unsqueeze(0).cuda().type(dtype=torch.float32)
boxes = torch.tensor([[1., 1., 5., 5.]]).cuda()/exp
ind = torch.tensor([0.]*len(boxes)).cuda().type(torch.float32)
y_exp, x_exp = fmap.shape[2:] # exp = expansion
boxes.mul_(torch.tensor([y_exp, x_exp, y_exp, x_exp], dtype=torch.float32).cuda())
boxes = torch.cat((ind.unsqueeze(1), boxes), dim=1)
aligned_tv = tv.ops.roi_align(fmap, boxes, output_size=pool_size, sampling_ratio=-1)
aligned = self.ra_ext.roi_align_2d(fmap, boxes, output_size=pool_size, sampling_ratio=-1)
boxes_3d = torch.cat((boxes, torch.tensor([[-1.,1.]]*len(boxes)).cuda()), dim=1)
fmap_3d = fmap.unsqueeze(dim=-1)
pool_size = (*pool_size,1)
ra_object = self.ra_ext.RoIAlign(output_size=pool_size, spatial_scale=1.,)
aligned_3d = ra_object(fmap_3d, boxes_3d)
expected_res = torch.tensor([[[[10.5000, 12.5000],
[22.5000, 24.5000]]]]).cuda()
expected_res_3d = torch.tensor([[[[[10.5000],[12.5000]],
[[22.5000],[24.5000]]]]]).cuda()
assert torch.all(aligned==expected_res), "2D RoIAlign check vs. specific example failed. res: {}\n expected: {}\n".format(aligned, expected_res)
assert torch.all(aligned_3d==expected_res_3d), "3D RoIAlign check vs. specific example failed. res: {}\n expected: {}\n".format(aligned_3d, expected_res_3d)
def build(self):
"""
Build Retina Net architecture.
"""
# Image size must be dividable by 2 multiple times.
h, w = self.cf.patch_size[:2]
if h / 2 ** 5 != int(h / 2 ** 5) or w / 2 ** 5 != int(w / 2 ** 5):
raise Exception("Image size must be dividable by 2 at least 5 times "
"to avoid fractions when downscaling and upscaling."
"For example, use 256, 320, 384, 448, 512, ... etc. ")
# instanciate abstract multi dimensional conv class and backbone model.
conv = mutils.NDConvGenerator(self.cf.dim)
backbone = utils.import_module('bbone', self.cf.backbone_path)
# build Anchors, FPN, Classifier / Bbox-Regressor -head
self.np_anchors = mutils.generate_pyramid_anchors(self.logger, self.cf)
self.anchors = torch.from_numpy(self.np_anchors).float().cuda()
# self.Fpn = backbone.FPN(self.cf, conv, operate_stride1=self.cf.operate_stride1)
self.num_classes=self.cf.head_classes
# i3d_rgb = backbone.I3D(num_classes=self.num_classes)
######## HEART OF I3D RGB Weight Transfer
# print("Transferring weight from i3d_rgb model...........")
# i3d_rgb = backbone.I3D(num_classes=self.num_classes)
# i3d_rgb.load_state_dict(torch.load('weight/model_rgb.pth'), strict=False)
# print('Success..........')
# self.Fpn = i3d_rgb
self.Fpn = backbone.I3D(num_classes=self.num_classes)
self.Classifier = Classifier(self.cf, conv)
self.BBRegressor = BBRegressor(self.cf, conv)
def test(self):
cf = utils.import_module("toy_cf", 'datasets/toy/configs.py').Configs()
exp_dir = "./unittesting/"
#checks = {"retina_net": False, "mrcnn": False}
#print("Testing for runtime errors with models {}".format(list(checks.keys())))
#for model in tqdm.tqdm(list(checks.keys())):
# cf.model = model
# cf.model_path = 'models/{}.py'.format(cf.model if not 'retina' in cf.model else 'retina_net')
# cf.model_path = os.path.join(cf.source_dir, cf.model_path)
# {'mrcnn': cf.add_mrcnn_configs,
# 'retina_net': cf.add_mrcnn_configs, 'retina_unet': cf.add_mrcnn_configs,
# 'detection_unet': cf.add_det_unet_configs, 'detection_fpn': cf.add_det_fpn_configs
# }[model]()
# todo change structure of configs-handling with exec.py so that its dynamically parseable instead of needing to
# todo be changed in the file all the time.
checks = {cf.model: False}
completed_process = subprocess.run(
"python exec.py --dev --dataset_name toy -m train_test --exp_dir {}"
.format(exp_dir),
shell=True,
capture_output=True,
text=True)
if completed_process.returncode != 0:
print("Runtime test of model {} failed due to\n{}".format(
cf.model, completed_process.stderr))
else:
checks[cf.model] = True
subprocess.call("rm -rf {}".format(exp_dir), shell=True)
assert all(checks.values()), "A runtime test crashed."
def test(self):
# dynamically import module so that it doesn't affect other tests if import fails
self.ra_ext = utils.import_module("ra_ext", 'custom_extensions/roi_align/roi_align.py')
# 2d test
self.check_2d()
# 3d test
self.check_3d()
return
def __init__(self, cf, logger):
super(net, self).__init__()
self.cf = cf
self.dim = cf.dim
self.norm = cf.norm
self.logger = logger
backbone = utils.import_module('bbone', cf.backbone_path)
self.c_gen = backbone.ConvGenerator(cf.dim)
self.Interpolator = backbone.Interpolate
#down = DownBlockGen(cf.dim)
#up = UpBlockGen(cf.dim, backbone.Interpolate)
down = self.down
up = self.up
pad = cf.pad
if pad=="same":
pad = (cf.kernel_size-1)//2
self.dims = "not yet recorded"
self.is_cuda = False
self.init = horiz_conv(len(cf.channels), cf.init_filts, cf.kernel_size, self.c_gen, self.norm, pad=pad,
relu=cf.relu)
self.down1 = down(cf.init_filts, cf.init_filts*2, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu)
self.down2 = down(cf.init_filts*2, cf.init_filts*4, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu)
self.down3 = down(cf.init_filts*4, cf.init_filts*6, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu)
self.down4 = down(cf.init_filts*6, cf.init_filts*8, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu,
maintain_z=True)
self.down5 = down(cf.init_filts*8, cf.init_filts*12, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu,
maintain_z=True)
#self.down6 = down(cf.init_filts*10, cf.init_filts*14, cf.kernel_size, cf.kernel_size_m, pad=pad, relu=cf.relu)
#self.up1 = up(cf.init_filts*14, cf.init_filts*10, cf.kernel_size, pad=pad, relu=cf.relu)
self.up2 = up(cf.init_filts*12, cf.init_filts*8, cf.kernel_size, pad=pad, relu=cf.relu, maintain_z=True)
self.up3 = up(cf.init_filts*8, cf.init_filts*6, cf.kernel_size, pad=pad, relu=cf.relu, maintain_z=True)
self.up4 = up(cf.init_filts*6, cf.init_filts*4, cf.kernel_size, pad=pad, relu=cf.relu)
self.up5 = up(cf.init_filts*4, cf.init_filts*2, cf.kernel_size, pad=pad, relu=cf.relu)
self.up6 = up(cf.init_filts*2, cf.init_filts, cf.kernel_size, pad=pad, relu=cf.relu)
self.seg = self.c_gen(cf.init_filts, cf.num_seg_classes, 1, norm=None, relu=None)
# initialize parameters
if self.cf.weight_init == "custom":
logger.info("Tried to use custom weight init which is not defined. Using pytorch default.")
elif self.cf.weight_init:
mutils.initialize_weights(self)
else:
logger.info("using default pytorch weight init")
def __init__(self, cf, logger):
super(net, self).__init__()
self.cf = cf
self.logger = logger
backbone = utils.import_module('bbone', cf.backbone_path)
conv = mutils.NDConvGenerator(cf.dim)
# set operate_stride1=True to generate a unet-like FPN.)
self.fpn = backbone.FPN(cf, conv, operate_stride1=True).cuda()
self.conv_final = conv(cf.end_filts, cf.num_seg_classes, ks=1, pad=0, norm=cf.norm, relu=None)
if self.cf.weight_init is not None:
logger.info("using pytorch weight init of type {}".format(self.cf.weight_init))
mutils.initialize_weights(self)
else:
logger.info("using default pytorch weight init")
from skimage.morphology import convex_hull_image
from scipy.ndimage.interpolation import zoom
from scipy.ndimage.morphology import binary_dilation, generate_binary_structure
LS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'lung-segmentation')
sys.path.append(LS_PATH)
import predict
from data import utils as data_utils
PROJECT_ROOT = Path(__file__).absolute().parent.parent.parent
sys.path.append(str(PROJECT_ROOT))
import utils.exp_utils as utils
dir_path = os.path.dirname(os.path.realpath(__file__))
cf_file = utils.import_module("cf", os.path.join(dir_path, "configs.py"))
cf = cf_file.configs()
# Load lung segmentation model from MLFlow
remote_server_uri = "http://mlflow.10.7.13.202.nip.io/"
mlflow.set_tracking_uri(remote_server_uri)
model_name = "2-lungs-segmentation"
unet = mlflow.pytorch.load_model(f"models:/{model_name}/production")
def resample_array(src_imgs, src_spacing, target_spacing):
src_spacing = np.round(src_spacing, 3)
target_shape = [
int(src_imgs.shape[ix] * src_spacing[::-1][ix] /
target_spacing[::-1][ix]) for ix in range(len(src_imgs.shape))
]
os.path.join(fold_dir, 'file_list.txt'), source_dir, target_dir),
shell=True)
n_threads = 8
dutils.unpack_dataset(target_dir, threads=n_threads)
copied_files = os.listdir(target_dir)
t = utils.get_formatted_duration(time.time() - start_time)
logger.info(
"\ncopying and unpacking data set finished using {} threads.\n{} files in target dir: {}. Took {}\n"
.format(n_threads, len(copied_files), target_dir, t))
if __name__ == "__main__":
total_stime = time.time()
cf_file = utils.import_module("cf", "configs.py")
cf = cf_file.configs()
cf.created_fold_id_pickle = False
cf.exp_dir = "dev/"
cf.plot_dir = cf.exp_dir + "plots"
os.makedirs(cf.exp_dir, exist_ok=True)
cf.fold = 0
logger = utils.get_logger(cf.exp_dir)
#batch_gen = get_train_generators(cf, logger)
#train_batch = next(batch_gen["train"])
test_gen = get_test_generator(cf, logger)
test_batch = next(test_gen["test"])
type=str,
default='experiments/toy_exp',
help=
'specifies, from which source experiment to load configs and data_loader.'
)
args = parser.parse_args()
folds = args.folds
resume_to_checkpoint = args.resume_to_checkpoint
if args.mode == 'train' or args.mode == 'train_test':
cf = utils.prep_exp(args.exp_source, args.exp_dir, args.server_env,
args.use_stored_settings)
cf.slurm_job_id = args.slurm_job_id
model = utils.import_module('model', cf.model_path)
data_loader = utils.import_module(
'dl', os.path.join(args.exp_source, 'data_loader.py'))
if folds is None:
folds = range(cf.n_cv_splits)
for fold in folds:
cf.fold_dir = os.path.join(cf.exp_dir, 'fold_{}'.format(fold))
cf.fold = fold
cf.resume_to_checkpoint = resume_to_checkpoint
if not os.path.exists(cf.fold_dir):
os.mkdir(cf.fold_dir)
logger = utils.get_logger(cf.fold_dir)
train(logger)
cf.resume_to_checkpoint = None
if args.mode == 'train_test':
#self.dataset_name = "datasets/prostate"
self.dataset_name = "datasets/lidc"
#self.exp_dir = "datasets/toy/experiments/mrcnnal2d_clkengal" # detunet2d_di_bs16_ps512"
#self.exp_dir = "/home/gregor/networkdrives/E132-Cluster-Projects/prostate/experiments/gs6071_retinau3d_cl_bs6"
#self.exp_dir = "/home/gregor/networkdrives/E132-Cluster-Projects/prostate/experiments/gs6071_frcnn3d_cl_bs6"
#self.exp_dir = "/home/gregor/networkdrives/E132-Cluster-Projects/prostate/experiments_t2/gs6071_mrcnn3d_cl_bs6_lessaug"
#self.exp_dir = "/home/gregor/networkdrives/E132-Cluster-Projects/prostate/experiments/gs6071_detfpn3d_cl_bs6"
#self.exp_dir = "/home/gregor/networkdrives/E132-Cluster-Projects/lidc_sa/experiments/ms12345_mrcnn3d_rgbin_bs8"
self.exp_dir = '/home/gregor/Documents/medicaldetectiontoolkit/datasets/lidc/experiments/ms12345_mrcnn3d_rg_bs8'
#self.exp_dir = '/home/gregor/Documents/medicaldetectiontoolkit/datasets/lidc/experiments/ms12345_mrcnn3d_rgbin_bs8'
self.server_env = False
args = Args()
data_loader = utils.import_module('dl', os.path.join(args.dataset_name, "data_loader.py"))
config_file = utils.import_module('cf', os.path.join(args.exp_dir, "configs.py"))
cf = config_file.Configs()
cf.exp_dir = args.exp_dir
cf.test_dir = cf.exp_dir
pid = '0811a'
cf.fold = find_pid_in_splits(pid)
#cf.fold = 0
cf.merge_2D_to_3D_preds = False
if cf.merge_2D_to_3D_preds:
cf.dim==3
cf.fold_dir = os.path.join(cf.exp_dir, 'fold_{}'.format(cf.fold))
anal_dir = os.path.join(cf.exp_dir, "inference_analysis")
cf = utils.prep_exp(args.dataset_name, args.exp_dir, args.server_env,
args.use_stored_settings)
if args.dev:
folds = [0, 1]
cf.batch_size, cf.num_epochs, cf.min_save_thresh, cf.save_n_models = 3 if cf.dim == 2 else 1, 2, 0, 2
cf.num_train_batches, cf.num_val_batches, cf.max_val_patients = 5, 1, 1
cf.test_n_epochs, cf.max_test_patients = cf.save_n_models, 2
torch.backends.cudnn.benchmark = cf.dim == 3
else:
torch.backends.cudnn.benchmark = cf.cuda_benchmark
if args.data_dest is not None:
cf.data_dest = args.data_dest
logger = utils.get_logger(cf.exp_dir, cf.server_env,
cf.sysmetrics_interval)
data_loader = utils.import_module(
'data_loader', os.path.join(args.dataset_name, 'data_loader.py'))
model = utils.import_module('model', cf.model_path)
logger.info("loaded model from {}".format(cf.model_path))
if folds is None:
folds = range(cf.n_cv_splits)
for fold in folds:
"""k-fold cross-validation: the dataset is split into k equally-sized folds, one used for validation,
one for testing, the rest for training. This loop iterates k-times over the dataset, cyclically moving the
splits. k==folds, fold in [0,folds) says which split is used for testing.
"""
cf.fold_dir = os.path.join(cf.exp_dir, 'fold_{}'.format(fold))
cf.fold = fold
logger.set_logfile(fold=fold)
cf.resume = resume
if not os.path.exists(cf.fold_dir):
args.use_stored_settings)
if args.dev:
folds = [0, 1]
cf.batch_size, cf.num_epochs, cf.min_save_thresh, cf.save_n_models = 3 if cf.dim == 2 else 1, 1, 0, 2
cf.num_train_batches, cf.num_val_batches, cf.max_val_patients = 5, 1, 1
cf.test_n_epochs = cf.save_n_models
cf.max_test_patients = 2
cf.data_dest = args.data_dest
logger = utils.get_logger(cf.exp_dir, cf.server_env)
logger.info("cudnn benchmark: {}, deterministic: {}.".format(
torch.backends.cudnn.benchmark,
torch.backends.cudnn.deterministic))
logger.info("sending tensors to CUDA device: {}.".format(
torch.cuda.get_device_name(args.cuda_device)))
data_loader = utils.import_module(
'dl', os.path.join(args.exp_source, 'data_loader.py'))
model = utils.import_module('mdt_model', cf.model_path)
logger.info("loaded model from {}".format(cf.model_path))
if folds is None:
folds = range(cf.n_cv_splits)
# MLFLow new experiment
try:
if args.mlflow_artifacts_uri is not None:
exp_id = mlflow.create_experiment(
args.mlflow_experiment_id,
artifact_location=args.mlflow_artifacts_uri)
else:
exp_id = mlflow.create_experiment(args.mlflow_experiment_id)
except:
exp_id = mlflow.set_experiment(args.mlflow_experiment_id)
def test(self):
print("Testing multithreaded iterator.")
dataset = "toy"
exp_dir = Path("datasets/{}/experiments/dev".format(dataset))
cf_file = utils.import_module("cf_file", exp_dir / "configs.py")
cf = cf_file.Configs()
dloader = utils.import_module(
'data_loader', 'datasets/{}/data_loader.py'.format(dataset))
cf.exp_dir = Path(exp_dir)
cf.n_workers = 5
cf.batch_size = 3
cf.fold = 0
cf.plot_dir = cf.exp_dir / "plots"
logger = utils.get_logger(cf.exp_dir, cf.server_env,
cf.sysmetrics_interval)
cf.num_val_batches = "all"
cf.val_mode = "val_sampling"
cf.n_workers = 8
batch_gens = dloader.get_train_generators(cf,
logger,
data_statistics=False)
val_loader = batch_gens["val_sampling"]
for epoch in range(4):
produced_ids = []
for i in range(batch_gens['n_val']):
batch = next(val_loader)
produced_ids.append(batch["pid"])
uni, cts = np.unique(np.concatenate(produced_ids),
return_counts=True)
assert np.all(
cts < 3
), "with batch size one: every item should occur exactly once.\n uni {}, cts {}".format(
uni[cts > 2], cts[cts > 2])
#assert len(np.setdiff1d(val_loader.generator.dataset_pids, uni))==0, "not all val pids were shown."
assert len(np.setdiff1d(uni, val_loader.generator.dataset_pids)
) == 0, "pids shown that are not val set. impossible?"
cf.n_workers = os.cpu_count()
cf.batch_size = int(
val_loader.generator.dataset_length / cf.n_workers) + 2
val_loader = dloader.create_data_gen_pipeline(
cf,
val_loader.generator._data,
do_aug=False,
sample_pids_w_replace=False,
max_batches=None,
raise_stop_iteration=True)
for epoch in range(2):
produced_ids = []
for b, batch in enumerate(val_loader):
produced_ids.append(batch["pid"])
uni, cts = np.unique(np.concatenate(produced_ids),
return_counts=True)
assert np.all(
cts == 1
), "with batch size one: every item should occur exactly once.\n uni {}, cts {}".format(
uni[cts > 1], cts[cts > 1])
assert len(np.setdiff1d(val_loader.generator.dataset_pids,
uni)) == 0, "not all val pids were shown."
assert len(np.setdiff1d(uni, val_loader.generator.dataset_pids)
) == 0, "pids shown that are not val set. impossible?"
pass