def main(dataset_type='cityscapes'): default_datasets, transformer_tag = \ dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, ('train', 'val')) # occlusion_counts_file = dataset_registry.REGISTRY['cityscapes'].get_occlusion_count_filename( # split='train', transformer_tag=transformer_tag) occlusion_counts_files = { split: dataset_registry.REGISTRY[dataset_type].get_occlusion_counts_filename( split, transformer_tag) for split in ('train', 'val') } semantic_class_names = default_datasets['train'].semantic_class_names occlusion_counts = { split: dataset_statistics.OcclusionsOfSameClass( range(len(semantic_class_names)), semantic_class_names=semantic_class_names, cache_file=occlusion_counts_files[split]) for split in ('train', 'val') } # occlusion_counts = { # split: dataset_statistics.OcclusionsOfSameClass( # range(semantic_class_names), semantic_class_names=semantic_class_names, # cache_file=occlusion_counts_files[split]) # for split in ('train', 'val') # } for split in ('train', 'val'): occlusion_counts[split].compute_or_retrieve(default_datasets[split]) for split in ('train', 'val'): write_stats(occlusion_counts[split].stat_tensor, split=split, semantic_class_names=semantic_class_names, metric_name='occlusion_counts')
def main(): # args, cfg_override_args = local_setup(loss) argv = sys.argv[1:] gpu = extract_from_argv(argv, key='--gpu', default_val=None) if gpu is None: gpu = extract_from_argv(argv, key='-g', default_val=[0]) config_idx = extract_from_argv(argv, key='-c', default_val='overfit_1') sampler_cfg = sampler_cfg_registry.get_sampler_cfg_set(sampler_arg=None) dataset_name = 'cityscapes' default_datasets, transformer_tag = get_default_datasets_for_instance_counts( dataset_name, ('train', )) cuda = torch.cuda.is_available() loader_kwargs = {'num_workers': 4, 'pin_memory': True} if cuda else {} batch_sizes = {split: 1 for split in default_datasets.keys()} samplers = sampler_factory.get_samplers(dataset_name, sampler_cfg, default_datasets, splits=default_datasets.keys()) dataloaders = { split: torch.utils.data.DataLoader(default_datasets[split], batch_size=batch_sizes[split], sampler=samplers[split], **loader_kwargs) for split in default_datasets.keys() } img, (sem_lbl, inst_lbl) = get_single_img_data(dataloaders['train'], 0) print('Got an image!')
def reduce_upsnet_to_image_set(ds, gt_files, test_logdir): train_cfg = yaml.load(open(os.path.join(test_logdir, 'config.yaml'), 'rb')) sampler_name = train_cfg['sampler'] if 'val' in test_logdir: split = 'val' else: raise Exception dataset_type = 'cityscapes' assert dataset_type in test_logdir, 'Are we still working with Cityscapes? Im assuming it in this code.' full_datasets, transformer_tag = get_default_datasets_for_instance_counts(dataset_type=dataset_type, splits=(split,)) full_dataset = full_datasets[split] sampler_cfg = sampler_cfg_registry.get_sampler_cfg_set(sampler_name) sampler_cfg = { k: v for k, v in sampler_cfg.items() if k == split } samplers = get_samplers(dataset_type=dataset_type, sampler_cfg=sampler_cfg, datasets={split: full_dataset}, splits=(split,)) upsnet_image_list = gt_files['upsnet']['images'] assert len(upsnet_image_list) == len(samplers[split].initial_indices) sampler_idxs = samplers[split].indices upsnet_reduced = ds['upsnet'] orig_collated_stats = ds['upsnet']['collated_stats_per_image_per_cat'].item() upsnet_reduced['collated_stats_per_image_per_cat'] = {k: v[sampler_idxs, :] for k, v in orig_collated_stats.items()} upsnet_reduced['reduced_idxs'] = sampler_idxs upsnet_reduced['reduction_sampler'] = (dataset_type, sampler_cfg) reduction_tag = 'sampler_{}_{}'.format(dataset_type, sampler_cfg) return upsnet_reduced, reduction_tag
def main(): datasets, transformer_tag = dataset_generator_registry.get_default_datasets_for_instance_counts( 'cityscapes', splits=('train', 'val')) dataset = datasets['train'] datapoint = dataset[0] print('id: ', datapoint['identifier']) image, sem_lbl, inst_lbl = datapoint['image'], datapoint[ 'sem_lbl'], datapoint['inst_lbl'] dataset_length = len(dataset) id_list = dataset.id_list
def get_instance_counts(dataset_type): default_datasets, transformer_tag = \ dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, splits=('train', 'val')) instance_counts = {} # Compute statistics for split in default_datasets.keys(): default_dataset = default_datasets[split] semantic_class_names = default_dataset.semantic_class_names instance_count_file = \ dataset_registry.REGISTRY[dataset_type].get_instance_count_filename(split, transformer_tag) instance_count_cache = dataset_statistics.NumberofInstancesPerSemanticClass( range(len(semantic_class_names)), cache_file=instance_count_file) instance_counts[split] = instance_count_cache.stat_tensor train_stats, val_stats = instance_counts['train'], instance_counts['val'] return train_stats, val_stats, default_train_dataset, default_val_dataset
def get_instance_counts(dataset_type): default_train_dataset, default_val_dataset, transformer_tag = \ dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type) default_datasets = { 'train': default_train_dataset, 'val': default_val_dataset, } # Compute statistics split = 'train' instance_count_file = os.path.join( dataset_registry.REGISTRY[dataset_type].dataset_path, '{}_instance_counts_{}.npy'.format(split, transformer_tag)) if not os.path.isfile(instance_count_file): print('Generating file {}'.format(instance_count_file)) stats = dataset_statistics.InstanceDatasetStatistics( default_datasets[split]) stats.compute_statistics( filename_to_write_instance_counts=instance_count_file) else: print( 'Reading from instance counts file {}'.format(instance_count_file)) stats = dataset_statistics.InstanceDatasetStatistics( default_datasets[split], existing_instance_count_file=instance_count_file) train_stats = stats split = 'val' instance_count_file = os.path.join( dataset_registry.REGISTRY[dataset_type].dataset_path, '{}_instance_counts_{}.npy'.format(split, transformer_tag)) if not os.path.isfile(instance_count_file): print('Generating file {}'.format(instance_count_file)) stats = dataset_statistics.InstanceDatasetStatistics( default_datasets[split]) stats.compute_statistics( filename_to_write_instance_counts=instance_count_file) else: print( 'Reading from instance counts file {}'.format(instance_count_file)) stats = dataset_statistics.InstanceDatasetStatistics( default_datasets[split], existing_instance_count_file=instance_count_file) val_stats = stats return train_stats, val_stats, default_train_dataset, default_val_dataset
def get_samplers(dataset_type, sampler_cfg, train_dataset, val_dataset): if sampler_cfg is None: train_sampler = sampler.sampler.RandomSampler(train_dataset) val_sampler = sampler.sampler.SequentialSampler(val_dataset) train_for_val_sampler = sampler.sampler.SequentialSampler( train_dataset) else: # Get 'clean' datasets for instance counting default_train_dataset, default_val_dataset, transformer_tag = \ dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type) # train sampler train_sampler = sampler_generator_helper(dataset_type, train_dataset, default_train_dataset, sampler_cfg, 'train', transformer_tag) # val sampler if isinstance(sampler_cfg['val'], str) and sampler_cfg['val'] == 'copy_train': val_sampler = train_sampler.copy(sequential_override=True) else: val_sampler = sampler_generator_helper(dataset_type, val_dataset, default_val_dataset, sampler_cfg, 'val', transformer_tag) # train_for_val sampler sampler_cfg['train_for_val'] = pop_without_del(sampler_cfg, 'train_for_val', None) cut_n_images = sampler_cfg['train_for_val'].n_images or len( train_dataset) train_for_val_sampler = train_sampler.copy( sequential_override=True, cut_n_images=None if cut_n_images is None else min( cut_n_images, len(train_sampler))) return train_sampler, val_sampler, train_for_val_sampler
def get_samplers(dataset_type, sampler_cfg, datasets, splits=('train', 'val', 'train_for_val')): samplers = {} if sampler_cfg is None: SamplerTypes = { 'train': sampler.sampler.RandomSampler, 'val': sampler.sampler.SequentialSampler, 'train_for_val': sampler.sampler.SequentialSampler } for split in splits: samplers[split] = SamplerTypes[split](datasets[split]) else: # Get 'clean' datasets for instance counting if not dataset_type == 'synthetic': dataset_splits = [s for s in splits if s != 'train_for_val'] default_datasets, transformer_tag = \ dataset_generator_registry.get_default_datasets_for_instance_counts(dataset_type, splits=dataset_splits) else: default_datasets = datasets transformer_tag = 'no_transformation' # train sampler if 'train' in splits: # Put 'train' first splits = ['train'] + [s for s in splits if s != 'train'] for split in splits: if split == 'val': if isinstance(sampler_cfg[split], str) and sampler_cfg[split] == 'copy_train': assert 'train' in splits, 'train must be in split if val needs to copy sampler from train' samplers[split] = samplers['train'].copy( sequential_override=True) else: samplers[split] = sampler_generator_helper( dataset_type, datasets[split], default_datasets[split], sampler_cfg, sampler_split_type=split, transformer_tag=transformer_tag) # train_for_val sampler elif split == 'train_for_val': assert 'train' in splits sampler_cfg['train_for_val'] = pop_without_del( sampler_cfg, 'train_for_val', None) cut_n_images = sampler_cfg['train_for_val'].n_images or len( datasets['train']) samplers['train_for_val'] = samplers['train'].copy( sequential_override=True, cut_n_images=None if cut_n_images is None else min( cut_n_images, len(samplers['train']))) else: samplers[split] = sampler_generator_helper( dataset_type, datasets[split], default_datasets[split], sampler_cfg, sampler_split_type=split, transformer_tag=transformer_tag) return samplers