def main(dataset_type='cityscapes'):
default_datasets, transformer_tag = \
dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, ('train', 'val'))
# occlusion_counts_file = dataset_registry.REGISTRY['cityscapes'].get_occlusion_count_filename(
# split='train', transformer_tag=transformer_tag)
occlusion_counts_files = {
split: dataset_registry.REGISTRY[dataset_type].get_occlusion_counts_filename(
split, transformer_tag)
for split in ('train', 'val')
}
semantic_class_names = default_datasets['train'].semantic_class_names
occlusion_counts = {
split: dataset_statistics.OcclusionsOfSameClass(
range(len(semantic_class_names)), semantic_class_names=semantic_class_names,
cache_file=occlusion_counts_files[split]) for split in ('train', 'val')
}
# occlusion_counts = {
# split: dataset_statistics.OcclusionsOfSameClass(
# range(semantic_class_names), semantic_class_names=semantic_class_names,
# cache_file=occlusion_counts_files[split])
# for split in ('train', 'val')
# }
for split in ('train', 'val'):
occlusion_counts[split].compute_or_retrieve(default_datasets[split])
for split in ('train', 'val'):
write_stats(occlusion_counts[split].stat_tensor, split=split,
semantic_class_names=semantic_class_names, metric_name='occlusion_counts')
def main():
# args, cfg_override_args = local_setup(loss)
argv = sys.argv[1:]
gpu = extract_from_argv(argv, key='--gpu', default_val=None)
if gpu is None:
gpu = extract_from_argv(argv, key='-g', default_val=[0])
config_idx = extract_from_argv(argv, key='-c', default_val='overfit_1')
sampler_cfg = sampler_cfg_registry.get_sampler_cfg_set(sampler_arg=None)
dataset_name = 'cityscapes'
default_datasets, transformer_tag = get_default_datasets_for_instance_counts(
dataset_name, ('train', ))
cuda = torch.cuda.is_available()
loader_kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {}
batch_sizes = {split: 1 for split in default_datasets.keys()}
samplers = sampler_factory.get_samplers(dataset_name,
sampler_cfg,
default_datasets,
splits=default_datasets.keys())
dataloaders = {
split: torch.utils.data.DataLoader(default_datasets[split],
batch_size=batch_sizes[split],
sampler=samplers[split],
**loader_kwargs)
for split in default_datasets.keys()
}
img, (sem_lbl, inst_lbl) = get_single_img_data(dataloaders['train'], 0)
print('Got an image!')
def reduce_upsnet_to_image_set(ds, gt_files, test_logdir):
train_cfg = yaml.load(open(os.path.join(test_logdir, 'config.yaml'), 'rb'))
sampler_name = train_cfg['sampler']
if 'val' in test_logdir:
split = 'val'
else:
raise Exception
dataset_type = 'cityscapes'
assert dataset_type in test_logdir, 'Are we still working with Cityscapes? Im assuming it in this code.'
full_datasets, transformer_tag = get_default_datasets_for_instance_counts(dataset_type=dataset_type,
splits=(split,))
full_dataset = full_datasets[split]
sampler_cfg = sampler_cfg_registry.get_sampler_cfg_set(sampler_name)
sampler_cfg = {
k: v for k, v in sampler_cfg.items() if k == split
}
samplers = get_samplers(dataset_type=dataset_type, sampler_cfg=sampler_cfg, datasets={split: full_dataset},
splits=(split,))
upsnet_image_list = gt_files['upsnet']['images']
assert len(upsnet_image_list) == len(samplers[split].initial_indices)
sampler_idxs = samplers[split].indices
upsnet_reduced = ds['upsnet']
orig_collated_stats = ds['upsnet']['collated_stats_per_image_per_cat'].item()
upsnet_reduced['collated_stats_per_image_per_cat'] = {k: v[sampler_idxs, :]
for k, v in orig_collated_stats.items()}
upsnet_reduced['reduced_idxs'] = sampler_idxs
upsnet_reduced['reduction_sampler'] = (dataset_type, sampler_cfg)
reduction_tag = 'sampler_{}_{}'.format(dataset_type, sampler_cfg)
return upsnet_reduced, reduction_tag
def main():
datasets, transformer_tag = dataset_generator_registry.get_default_datasets_for_instance_counts(
'cityscapes', splits=('train', 'val'))
dataset = datasets['train']
datapoint = dataset[0]
print('id: ', datapoint['identifier'])
image, sem_lbl, inst_lbl = datapoint['image'], datapoint[
'sem_lbl'], datapoint['inst_lbl']
dataset_length = len(dataset)
id_list = dataset.id_list
def get_instance_counts(dataset_type):
default_datasets, transformer_tag = \
dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, splits=('train', 'val'))
instance_counts = {}
# Compute statistics
for split in default_datasets.keys():
default_dataset = default_datasets[split]
semantic_class_names = default_dataset.semantic_class_names
instance_count_file = \
dataset_registry.REGISTRY[dataset_type].get_instance_count_filename(split,
transformer_tag)
instance_count_cache = dataset_statistics.NumberofInstancesPerSemanticClass(
range(len(semantic_class_names)), cache_file=instance_count_file)
instance_counts[split] = instance_count_cache.stat_tensor
train_stats, val_stats = instance_counts['train'], instance_counts['val']
return train_stats, val_stats, default_train_dataset, default_val_dataset
def get_instance_counts(dataset_type):
default_train_dataset, default_val_dataset, transformer_tag = \
dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type)
default_datasets = {
'train': default_train_dataset,
'val': default_val_dataset,
}
# Compute statistics
split = 'train'
instance_count_file = os.path.join(
dataset_registry.REGISTRY[dataset_type].dataset_path,
'{}_instance_counts_{}.npy'.format(split, transformer_tag))
if not os.path.isfile(instance_count_file):
print('Generating file {}'.format(instance_count_file))
stats = dataset_statistics.InstanceDatasetStatistics(
default_datasets[split])
stats.compute_statistics(
filename_to_write_instance_counts=instance_count_file)
else:
print(
'Reading from instance counts file {}'.format(instance_count_file))
stats = dataset_statistics.InstanceDatasetStatistics(
default_datasets[split],
existing_instance_count_file=instance_count_file)
train_stats = stats
split = 'val'
instance_count_file = os.path.join(
dataset_registry.REGISTRY[dataset_type].dataset_path,
'{}_instance_counts_{}.npy'.format(split, transformer_tag))
if not os.path.isfile(instance_count_file):
print('Generating file {}'.format(instance_count_file))
stats = dataset_statistics.InstanceDatasetStatistics(
default_datasets[split])
stats.compute_statistics(
filename_to_write_instance_counts=instance_count_file)
else:
print(
'Reading from instance counts file {}'.format(instance_count_file))
stats = dataset_statistics.InstanceDatasetStatistics(
default_datasets[split],
existing_instance_count_file=instance_count_file)
val_stats = stats
return train_stats, val_stats, default_train_dataset, default_val_dataset
def get_samplers(dataset_type, sampler_cfg, train_dataset, val_dataset):
if sampler_cfg is None:
train_sampler = sampler.sampler.RandomSampler(train_dataset)
val_sampler = sampler.sampler.SequentialSampler(val_dataset)
train_for_val_sampler = sampler.sampler.SequentialSampler(
train_dataset)
else:
# Get 'clean' datasets for instance counting
default_train_dataset, default_val_dataset, transformer_tag = \
dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type)
# train sampler
train_sampler = sampler_generator_helper(dataset_type, train_dataset,
default_train_dataset,
sampler_cfg, 'train',
transformer_tag)
# val sampler
if isinstance(sampler_cfg['val'],
str) and sampler_cfg['val'] == 'copy_train':
val_sampler = train_sampler.copy(sequential_override=True)
else:
val_sampler = sampler_generator_helper(dataset_type, val_dataset,
default_val_dataset,
sampler_cfg, 'val',
transformer_tag)
# train_for_val sampler
sampler_cfg['train_for_val'] = pop_without_del(sampler_cfg,
'train_for_val', None)
cut_n_images = sampler_cfg['train_for_val'].n_images or len(
train_dataset)
train_for_val_sampler = train_sampler.copy(
sequential_override=True,
cut_n_images=None if cut_n_images is None else min(
cut_n_images, len(train_sampler)))
return train_sampler, val_sampler, train_for_val_sampler
def get_samplers(dataset_type,
sampler_cfg,
datasets,
splits=('train', 'val', 'train_for_val')):
samplers = {}
if sampler_cfg is None:
SamplerTypes = {
'train': sampler.sampler.RandomSampler,
'val': sampler.sampler.SequentialSampler,
'train_for_val': sampler.sampler.SequentialSampler
}
for split in splits:
samplers[split] = SamplerTypes[split](datasets[split])
else:
# Get 'clean' datasets for instance counting
if not dataset_type == 'synthetic':
dataset_splits = [s for s in splits if s != 'train_for_val']
default_datasets, transformer_tag = \
dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, splits=dataset_splits)
else:
default_datasets = datasets
transformer_tag = 'no_transformation'
# train sampler
if 'train' in splits: # Put 'train' first
splits = ['train'] + [s for s in splits if s != 'train']
for split in splits:
if split == 'val':
if isinstance(sampler_cfg[split],
str) and sampler_cfg[split] == 'copy_train':
assert 'train' in splits, 'train must be in split if val needs to copy sampler from train'
samplers[split] = samplers['train'].copy(
sequential_override=True)
else:
samplers[split] = sampler_generator_helper(
dataset_type,
datasets[split],
default_datasets[split],
sampler_cfg,
sampler_split_type=split,
transformer_tag=transformer_tag)
# train_for_val sampler
elif split == 'train_for_val':
assert 'train' in splits
sampler_cfg['train_for_val'] = pop_without_del(
sampler_cfg, 'train_for_val', None)
cut_n_images = sampler_cfg['train_for_val'].n_images or len(
datasets['train'])
samplers['train_for_val'] = samplers['train'].copy(
sequential_override=True,
cut_n_images=None if cut_n_images is None else min(
cut_n_images, len(samplers['train'])))
else:
samplers[split] = sampler_generator_helper(
dataset_type,
datasets[split],
default_datasets[split],
sampler_cfg,
sampler_split_type=split,
transformer_tag=transformer_tag)
return samplers