def get_dataset(args: argparse.Namespace) -> torch.utils.data.Dataset:
"""Returns dataset.
Returns
-------
torch.utils.data.Dataset
Dataset object.
"""
if sum([args.dataset is not None, args.path_tif is not None]) != 1:
raise ValueError("Must specify one input source type")
if args.dataset is not None:
ds_fn = str_to_object(args.dataset)
if not isinstance(ds_fn, Callable):
raise ValueError(f"{args.dataset} must be callable")
return ds_fn(**args.dataset_kwargs)
if args.path_tif is not None:
if not os.path.exists(args.path_tif):
raise ValueError(f"Path does not exists: {args.path_tif}")
paths_tif = [args.path_tif]
if os.path.isdir(args.path_tif):
paths_tif = files_from_dir(args.path_tif)
ds = TiffDataset(
dataframe=pd.DataFrame({
"path_bf": paths_tif,
"path_target": None
}),
transform_signal=[norm_around_center],
transform_target=[norm_around_center],
col_signal="path_bf",
)
return ds
raise NotImplementedError
def __init__(
self,
betas=(0.5, 0.999),
criterion_class='fnet.losses.WeightedMSE',
init_weights=True,
lr=0.001,
nn_class='fnet.nn_modules.fnet_nn_3d.Net',
nn_kwargs={},
scheduler=None,
weight_decay=0,
gpu_ids=-1,
):
self.betas = betas
self.criterion = str_to_object(criterion_class)()
self.gpu_ids = [gpu_ids] if isinstance(gpu_ids, int) else gpu_ids
self.init_weights = init_weights
self.lr = lr
self.nn_class = nn_class
self.nn_kwargs = nn_kwargs
self.scheduler = scheduler
self.weight_decay = weight_decay
self.count_iter = 0
self.device = (torch.device('cuda', self.gpu_ids[0])
if self.gpu_ids[0] >= 0 else torch.device('cpu'))
self.optimizer = None
self._init_model()
self.fnet_model_kwargs, self.fnet_model_posargs = get_args()
self.fnet_model_kwargs.pop('self')
def _init_model(self):
self.net = str_to_object(self.nn_class)(**self.nn_kwargs)
if self.init_weights:
self.net.apply(_weights_init)
self.net.to(self.device)
self.optimizer = torch.optim.Adam(
get_per_param_options(self.net, wd=self.weight_decay),
lr=self.lr,
betas=self.betas,
)
if self.scheduler is not None:
if self.scheduler[0] == 'snapshot':
period = self.scheduler[1]
self.scheduler = torch.optim.lr_scheduler.LambdaLR(
self.optimizer,
lambda x: (0.01 + (1 - 0.01) *
(0.5 + 0.5 * math.cos(math.pi *
(x % period) / period))),
)
elif self.scheduler[0] == 'step':
step_size = self.scheduler[1]
self.scheduler = torch.optim.lr_scheduler.StepLR(
self.optimizer, step_size)
else:
raise NotImplementedError
def get_bpds_train(args: argparse.Namespace) -> BufferedPatchDataset:
"""Creates data provider for training."""
ds_fn = str_to_object(args.dataset_train)
if not isinstance(ds_fn, Callable):
raise ValueError('Dataset function should be Callable')
ds = ds_fn(**args.dataset_train_kwargs)
return BufferedPatchDataset(dataset=ds, **args.bpds_kwargs)
def main(args: Optional[argparse.Namespace] = None) -> None:
"""Predicts using model."""
if args is None:
parser = argparse.ArgumentParser()
add_parser_arguments(parser)
args = parser.parse_args()
if args.json and not args.json.exists():
save_default_predict_options(args.json)
return
load_from_json(args)
metric = str_to_object(args.metric)
dataset = get_dataset(args)
entries = []
model = None
indices = get_indices(args, dataset)
for count, idx in enumerate(indices, 1):
logger.info(f"Processing: {idx:3d} ({count}/{len(indices)})")
entry = {}
entry["index"] = idx
signal, target = item_from_dataset(dataset, idx)
if not args.no_signal:
entry["path_signal"] = save_tif(f"{idx}_signal.tif",
signal.numpy()[0, ],
args.path_save_dir)
if not args.no_target and target is not None:
entry["path_target"] = save_tif(f"{idx}_target.tif",
target.numpy()[0, ],
args.path_save_dir)
for path_model_dir in args.path_model_dir:
if model is None or len(args.path_model_dir) > 1:
model_def = parse_model(path_model_dir)
model = load_model(model_def["path"], no_optim=True)
model.to_gpu(args.gpu_ids)
logger.info(f'Loaded model: {model_def["name"]}')
prediction = model.predict_piecewise(
signal, tta=("no_tta" not in model_def["options"]))
# signal = to_numpy(signal)
# print(signal.shape)
# print(signal.shape[1])
# network = model.net
# network.eval()
# with torch.no_grad():
# prediction = predict_on_zslice_tiles(network, signal)
evaluation = metric(target, prediction)
entry[args.metric + f'.{model_def["name"]}'] = evaluation
if not args.no_prediction:
for idx_c in range(prediction.size()[0]):
tag = f'prediction_c{idx_c}.{model_def["name"]}'
pred_c = prediction.numpy()[idx_c, ]
entry[f"path_{tag}"] = save_tif(f"{idx}_{tag}.tif", pred_c,
args.path_save_dir)
entries.append(entry)
save_predictions_csv(
path_csv=os.path.join(args.path_save_dir, "predictions.csv"),
pred_records=entries,
dataset=dataset,
)
save_args_as_json(args.path_save_dir, args)
def main(args: Optional[argparse.Namespace] = None) -> None:
"""Predicts using model."""
if args is None:
parser = argparse.ArgumentParser()
add_parser_arguments(parser)
args = parser.parse_args()
metric = str_to_object(args.metric)
dataset = get_dataset(args)
index_name = dataset.df.index.name or 'index'
if not os.path.exists(args.path_save_dir):
os.makedirs(args.path_save_dir)
entries = []
model = None
indices = (args.idx_sel if args.idx_sel is not None else
dataset.df.index)[:args.n_images if args.n_images > 0 else None]
for count, idx in enumerate(indices, 1):
print(f'Processing: {idx:3d} ({count}/{len(indices)})')
entry = {}
entry[index_name] = idx
data = dataset.loc[idx]
signal = data[0]
target = data[1] if len(data) > 1 else None
if not args.no_signal:
entry['path_signal'] = save_tif(f'{idx}_signal.tif',
signal.numpy()[0, ],
args.path_save_dir)
if not args.no_target and target is not None:
entry['path_target'] = save_tif(f'{idx}_target.tif',
target.numpy()[0, ],
args.path_save_dir)
for path_model_dir in args.path_model_dir:
if model is None or len(args.path_model_dir) > 1:
model_def = parse_model(path_model_dir)
model = load_model(model_def['path'], no_optim=True)
model.to_gpu(args.gpu_ids)
print('Predicting with:', model_def['name'])
prediction = model.predict_piecewise(
signal,
tta=('no_tta' not in model_def['options']),
)
if args.add_sigmoid:
prediction = torch.nn.functional.sigmoid(prediction)
evaluation = metric(target, prediction)
entry[args.metric + f'.{model_def["name"]}'] = evaluation
if not args.no_prediction and prediction is not None:
for idx_c in range(prediction.size()[0]):
tag = f'prediction_c{idx_c}.{model_def["name"]}'
pred_c = prediction.numpy()[idx_c, ]
entry[f'path_{tag}'] = save_tif(f'{idx}_{tag}.tif', pred_c,
args.path_save_dir)
entries.append(entry)
if ((count % 8) == 0) or (idx == indices[-1]):
save_csv(
os.path.join(args.path_save_dir, 'predictions.csv'),
pd.DataFrame(entries).set_index(index_name),
)
save_args_as_json(args.path_save_dir, args)
def get_bpds_val(args: argparse.Namespace) -> Optional[BufferedPatchDataset]:
"""Creates data provider for validation."""
if args.dataset_val is None:
return None
bpds_kwargs = copy.deepcopy(args.bpds_kwargs)
ds_fn = str_to_object(args.dataset_val)
if not isinstance(ds_fn, Callable):
raise ValueError('Dataset function should be Callable')
ds = ds_fn(**args.dataset_val_kwargs)
bpds_kwargs['buffer_size'] = min(4, len(ds))
bpds_kwargs['buffer_switch_interval'] = -1
return BufferedPatchDataset(dataset=ds, **bpds_kwargs)
def main(args: Optional[argparse.Namespace] = None) -> None:
"""Predicts using model."""
if args is None:
parser = argparse.ArgumentParser()
add_parser_arguments(parser)
args = parser.parse_args()
if args.json and not args.json.exists():
save_default_predict_options(args.json)
return
load_from_json(args)
metric = str_to_object(args.metric)
dataset = get_dataset(args)
entries = []
model = None
indices = get_indices(args, dataset)
for count, idx in enumerate(indices, 1):
logger.info(f'Processing: {idx:3d} ({count}/{len(indices)})')
entry = {}
entry['index'] = idx
signal, target = item_from_dataset(dataset, idx)
if not args.no_signal:
entry['path_signal'] = save_tif(f'{idx}_signal.tif',
signal.numpy()[0, ],
args.path_save_dir)
if not args.no_target and target is not None:
entry['path_target'] = save_tif(f'{idx}_target.tif',
target.numpy()[0, ],
args.path_save_dir)
for path_model_dir in args.path_model_dir:
if model is None or len(args.path_model_dir) > 1:
model_def = parse_model(path_model_dir)
model = load_model(model_def['path'], no_optim=True)
model.to_gpu(args.gpu_ids)
logger.info(f'Loaded model: {model_def["name"]}')
prediction = model.predict_piecewise(
signal,
tta=('no_tta' not in model_def['options']),
)
evaluation = metric(target, prediction)
entry[args.metric + f'.{model_def["name"]}'] = evaluation
if not args.no_prediction:
for idx_c in range(prediction.size()[0]):
tag = f'prediction_c{idx_c}.{model_def["name"]}'
pred_c = prediction.numpy()[idx_c, ]
entry[f'path_{tag}'] = save_tif(f'{idx}_{tag}.tif', pred_c,
args.path_save_dir)
entries.append(entry)
save_predictions_csv(
path_csv=os.path.join(args.path_save_dir, 'predictions.csv'),
pred_records=entries,
dataset=dataset,
)
save_args_as_json(args.path_save_dir, args)
def get_dataloader_train(
args: argparse.Namespace,
n_iter_remaining: int,
) -> torch.utils.data.DataLoader:
"""Creates DataLoader for training."""
bpds_kwargs = copy.deepcopy(args.bpds_kwargs)
bpds_kwargs['npatches'] = n_iter_remaining*args.batch_size
ds_fn = str_to_object(args.dataset_train)
if not isinstance(ds_fn, Callable):
raise ValueError('Dataset function should be Callable')
ds = ds_fn(**args.dataset_train_kwargs)
bpds = fnet.data.BufferedPatchDataset(dataset=ds, **bpds_kwargs)
dataloader = torch.utils.data.DataLoader(bpds, batch_size=args.batch_size)
return dataloader
def get_dataloader_val(
args: argparse.Namespace) -> Optional[torch.utils.data.DataLoader]:
"""Creates DataLoader for validation."""
if args.dataset_val is None:
return None
bpds_kwargs = copy.deepcopy(args.bpds_kwargs)
ds_fn = str_to_object(args.dataset_val)
if not isinstance(ds_fn, Callable):
raise ValueError('Dataset function should be Callable')
ds = ds_fn(**args.dataset_val_kwargs)
bpds_kwargs['buffer_size'] = min(4, len(ds))
bpds_kwargs['buffer_switch_frequency'] = -1
bpds_kwargs['npatches'] = 16 * args.batch_size
bpds = fnet.data.BufferedPatchDataset(dataset=ds, **bpds_kwargs)
dataloader = torch.utils.data.DataLoader(bpds, batch_size=args.batch_size)
return dataloader
def test_str_to_object():
exp = [_dummy, random.randrange]
for idx_s, as_str in enumerate(['_dummy', 'random.randrange']):
obj = str_to_object(as_str)
assert obj is exp[idx_s], f'{obj} is not {exp[idx_s]}'
def test_str_to_object():
"""Test string-to-object conversion."""
exp = [_dummy, random.randrange]
for idx_s, as_str in enumerate(['_dummy', 'random.randrange']):
obj = str_to_object(as_str)
assert obj is exp[idx_s], f'{obj} is not {exp[idx_s]}'
