def get_dataset_split(config):
ft = config['split'].lower().strip()
testf = try_parse(config['test'])
trainf = try_parse(config['train'])
validf = try_parse(config['valid'])
if ft == 'train_test_split':
seed = try_parse_or_default(config, 'seed', 42)
def split(images):
tv, test = train_test_split(images, train_size=1-testf, test_size=testf, random_state=seed)
train, valid = train_test_split(tv, train_size=trainf, test_size=validf, random_state=seed)
return {dp.TRAIN:train, dp.VALID:valid, dp.TEST:test}
elif ft == 'filter':
def split(images):
train = np.array(list(filter(lambda x: x.find(trainf) != -1, images)))
valid = np.array(list(filter(lambda x: x.find(validf) != -1, images)))
test = np.array(list(filter(lambda x: x.find(testf) != -1, images)))
return {dp.TRAIN:train, dp.VALID:valid, dp.TEST:test}
else:
raise ValueError(f"Unknown filter type: {ft}")
return split
def get_optimizer(config):
config = deepcopy(config)
name = config['name'].strip()
del config['name']
props = {k: try_parse(v) for k, v in config.items()}
try:
opt_class = getattr(optimizers, name)
opt = opt_class(**props)
return opt
except:
raise ValueError(f"Unknown optimizer: {name}")
def get_input_mapping(config):
config = deepcopy(config)
mt = config['mapping'].lower().strip()
del config['mapping']
props = {k: try_parse(v) for k, v in config.items()}
try:
func = getattr(dp, mt)
f = lambda dataset: func(dataset, **props)
except:
raise ValueError(f"Unknown dataset mapping: {mt}")
return f
def get_metrics(config):
def load_metric(name):
try:
metric_cls = getattr(metrics, name)
metric = metric_cls() if inspect.isclass(metric_cls) else metric_cls
return metric
except:
raise ValueError(f"Unknown metric: {name}")
names = try_parse(config['names'].strip())
if type(names) is str:
return [load_metric(names)]
else:
return list(map(lambda x: load_metric(x), names))
def get_dataset_mapping(config):
config = deepcopy(config)
mt = config['mapping'].lower().strip()
del config['mapping']
props = {k: try_parse(v) for k, v in config.items()}
if mt == 'image_histogram_mapping_segmentation':
f = lambda *x: dp.image_histogram_mapping_segmentation(*x, **props)
elif mt == 'image_histogram_mapping':
f = lambda *x: dp.image_histogram_mapping(*x, **props)
else:
try:
func = getattr(dp, mt)
f = lambda *x: func(*x, **props)
except:
raise ValueError(f"Unknown dataset mapping: {mt}")
return f
def get_loss(config):
config = deepcopy(config)
loss_name = config['name'].strip()
del config['name']
loss_weight = try_parse_or_default(config, "weight", default=1.0)
try:
del config['weight']
except:
pass
props = {k: try_parse(v) for k, v in config.items()}
try:
loss_cls = getattr(losses, loss_name)
loss = loss_cls(**props)
return loss, loss_weight
except:
raise ValueError(f"Unknown loss: {loss_name}")
def get_parameters(config):
return {k.lower().strip(): try_parse(v) for k, v in config.items()}
def get_dataset(config,type_, paths_filter=lambda x: x, out_mapping=lambda *x: x):
dt = config['dataset_type'].lower().strip()
dataset_path = config['dataset_path'].strip()
batch_size = try_parse_or_default(config, 'batch_size', 1)
size = try_parse(config['size'])
if dt == "cube2":
regression = try_parse_or_default(config, 'regression', False)
return_gt = try_parse_or_default(config, 'return_gt', True)
return_gt_mask = try_parse_or_default(config, 'return_gt_mask', False)
round_mask = try_parse_or_default(config, 'round', True)
gamma = try_parse_or_default(config, 'gamma', False)
resample = try_parse_or_default(config, 'resample', 1)
camera = try_parse_or_default(config, 'camera', None)
scene_type = try_parse_or_default(config, 'scene_type', None)
im_paths = paths_filter(Cube2Dataset.load_paths(dataset_path, config['list_name']))
ds = Cube2Dataset.dataset(im_paths, type_, batch_size, cache=False,
regression=regression, gt=return_gt, gt_mask=return_gt_mask,
round=round_mask, gamma=gamma, resample=resample,
camera=camera, scene_type=scene_type,
h=size[0], w=size[1], map_fn=out_mapping)
if type(ds) == tuple:
ds, im_paths = ds
elif dt == "newcube2":
regression = try_parse_or_default(config, 'regression', False)
return_gt = try_parse_or_default(config, 'return_gt', True)
return_gt_mask = try_parse_or_default(config, 'return_gt_mask', False)
round_mask = try_parse_or_default(config, 'round', True)
gamma = try_parse_or_default(config, 'gamma', False)
resample = try_parse_or_default(config, 'resample', 1)
camera = try_parse_or_default(config, 'camera', None)
scene_type = try_parse_or_default(config, 'scene_type', None)
im_paths = paths_filter(Cube2Dataset.load_paths(dataset_path, config['list_name']))
ds = NewCube2Dataset.dataset(im_paths, type_, batch_size, cache=False,
regression=regression, gt=return_gt, gt_mask=return_gt_mask,
round=round_mask, gamma=gamma, resample=resample,
camera=camera, scene_type=scene_type,
h=size[0], w=size[1], map_fn=out_mapping)
if type(ds) == tuple:
ds, im_paths = ds
elif dt == "cuben":
regression = try_parse_or_default(config, 'regression', False)
return_gt = try_parse_or_default(config, 'return_gt', True)
round_mask = try_parse_or_default(config, 'round', True)
gamma = try_parse_or_default(config, 'gamma', False)
n = try_parse_or_default(config, 'n', 3)
im_paths = paths_filter(Cube2Dataset.load_paths(dataset_path, config['list_name']))
ds = CubeNDataset.dataset(im_paths, type_, batch_size, cache=False,
regression=regression, gt=return_gt, round=round_mask, gamma=gamma,
h=size[0], w=size[1], n=n, map_fn=out_mapping)
elif dt == 'cube':
im_paths = paths_filter(CubeDataset.load_image_names(config['list_name'], dataset_path))
gts = np.loadtxt(os.path.join(dataset_path, config['gts_name']))
indices = np.array(list(map(lambda x: int(x[x.rfind('/') + 1:-4]) - 1, im_paths)))
ds = CubeDataset.regression_dataset(im_paths, indices, type=type_,
bs=batch_size, cache=False, uv=False,
gts=gts, map_fn=out_mapping, sz=size)
elif dt == 'tau':
im_paths = np.loadtxt(os.path.join(dataset_path, config['list_name']), dtype=str)
im_paths = paths_filter(im_paths)
im_paths = list(map(lambda x: os.path.join(dataset_path, x[1:]), im_paths))
ds = TauDataset.regression_dataset(im_paths, type=type_, bs=batch_size,
cache=False, uv=False,
sz=size, map_fn=out_mapping)
else:
raise ValueError(f"Unknown dataset type: {dt}")
return ds, len(im_paths)