def test_pytorch_iterator_last_batch_pad_last_batch(): num_gpus = 1 batch_size = 100 iters = 0 pipes, data_size = create_pipeline(lambda gpu: COCOReaderPipeline(batch_size=batch_size, num_threads=4, shard_id=gpu, num_gpus=num_gpus, data_paths=data_sets[0], random_shuffle=True, stick_to_shard=False, shuffle_after_epoch=False, pad_last_batch=True), batch_size, num_gpus) dali_train_iter = PyTorchIterator(pipes, output_map=["data"], size=pipes[0].epoch_size("Reader"), fill_last_batch=True) img_ids_list, img_ids_list_set, mirrored_data, _, _ = \ gather_ids(dali_train_iter, lambda x: x["data"].squeeze().numpy(), lambda x: 0, data_size) assert len(img_ids_list) > data_size assert len(img_ids_list_set) == data_size assert len(set(mirrored_data)) == 1 dali_train_iter.reset() next_img_ids_list, next_img_ids_list_set, next_mirrored_data, _, _ = \ gather_ids(dali_train_iter, lambda x: x["data"].squeeze().numpy(), lambda x: 0, data_size) assert len(next_img_ids_list) > data_size assert len(next_img_ids_list_set) == data_size assert len(set(next_mirrored_data)) == 1
def test_pytorch_iterator_not_fill_last_batch_pad_last_batch(): from nvidia.dali.plugin.pytorch import DALIGenericIterator as PyTorchIterator num_gpus = 1 batch_size = 100 pipes, data_size = create_pipeline(lambda gpu: COCOReaderPipeline(batch_size=batch_size, num_threads=4, shard_id=gpu, num_gpus=num_gpus, data_paths=data_sets[0], random_shuffle=False, stick_to_shard=False, shuffle_after_epoch=False, pad_last_batch=True), batch_size, num_gpus) dali_train_iter = PyTorchIterator(pipes, output_map=["data"], size=pipes[0].epoch_size("Reader"), fill_last_batch=False, last_batch_padded=True) img_ids_list, img_ids_list_set, mirrored_data, _, _ = \ gather_ids(dali_train_iter, lambda x: x["data"].squeeze().numpy(), lambda x: 0, data_size) assert len(img_ids_list) == data_size assert len(img_ids_list_set) == data_size assert len(set(mirrored_data)) != 1 dali_train_iter.reset() next_img_ids_list, next_img_ids_list_set, next_mirrored_data, _, _ = \ gather_ids(dali_train_iter, lambda x: x["data"].squeeze().numpy(), lambda x: 0, data_size) # there is no mirroring as data in the output is just cut off, # in the mirrored_data there is real data assert len(next_img_ids_list) == data_size assert len(next_img_ids_list_set) == data_size assert len(set(next_mirrored_data)) != 1
def update(self, resolution_level, batch_size): ''' :param resolution_level: resolution_level: 2-2,3-8,4-16,5-32,6-64,7-128,8-256 :return: ''' assert resolution_level >= 2 and resolution_level <= 8, "res error" self.batchsize = int(self.batch_table[pow(2, resolution_level)]) self.pipeline = TFRecordPipeline( batch_size=batch_size, size=pow(2, resolution_level), num_threads=4, device_id=self.device_idx, path= "/home/guyuchao/ssd/dataset/lsun-master/lsun_torch_tfrecord2/-r%02d.tfrecords" % resolution_level, index_path= "/home/guyuchao/ssd/dataset/lsun-master/lsun_torch_tfrecord2/-r%02d.idx" % resolution_level) self.pipeline.build() self.dali_iter = DALIGenericIterator( [self.pipeline], ["image/encoded"], self.pipeline.epoch_size("Reader"), auto_reset=True)
def __init__(self, dali_pipelines, transcripts, tokenizer, batch_size, shard_size, pipeline_type, normalize_transcripts=False): self.normalize_transcripts = normalize_transcripts self.tokenizer = tokenizer self.batch_size = batch_size from nvidia.dali.plugin.pytorch import DALIGenericIterator from nvidia.dali.plugin.base_iterator import LastBatchPolicy # in train pipeline shard_size is set to divisable by batch_size, so PARTIAL policy is safe if pipeline_type == 'val': self.dali_it = DALIGenericIterator( dali_pipelines, ["audio", "label", "audio_shape"], reader_name="Reader", dynamic_shape=True, auto_reset=True, last_batch_policy=LastBatchPolicy.PARTIAL) else: self.dali_it = DALIGenericIterator( dali_pipelines, ["audio", "label", "audio_shape"], size=shard_size, dynamic_shape=True, auto_reset=True, last_batch_padded=True, last_batch_policy=LastBatchPolicy.PARTIAL) self.tokenize(transcripts)
def __init__(self, dali_pipelines, transcripts, symbols, batch_size, reader_name, train_iterator: bool): self.transcripts = transcripts self.symbols = symbols self.batch_size = batch_size # in train pipeline shard_size is set to divisable by batch_size, # so PARTIAL policy is safe self.dali_it = DALIGenericIterator( dali_pipelines, ["audio", "label", "audio_shape"], reader_name=reader_name, dynamic_shape=True, auto_reset=True, last_batch_policy=LastBatchPolicy.DROP)
def __init__(self, dali_pipelines, transcripts, symbols, batch_size, reader_name, train_iterator: bool): self.transcripts = transcripts self.symbols = symbols self.batch_size = batch_size from nvidia.dali.plugin.pytorch import DALIGenericIterator from nvidia.dali.plugin.base_iterator import LastBatchPolicy self.dali_it = DALIGenericIterator( dali_pipelines, ["audio", "label", "audio_shape"], reader_name=reader_name, dynamic_shape=True, auto_reset=True, last_batch_policy=(LastBatchPolicy.DROP if train_iterator else LastBatchPolicy.PARTIAL))
def dali_makes_miricle(): it = SegInputIterator(4, 0, 1) pipe = panopticPipeline(it, 4, 1, 0) train_loader = DALIGenericIterator(pipe, ["images", "anns"], fill_last_batch=True) for nbatch, data in enumerate(train_loader): images = data[0]["images"] anns = data[0]["anns"] break print(images) edges = torch.zeros_like(anns) neighbour = torch.cat((anns[:, 1:], anns[:, -1:]), 1) edges += (anns != (0 or neighbour)).type(torch.uint8) neighbour = torch.cat((anns[:, :1], anns[:, :-1]), 1) edges += (anns != (0 or neighbour)).type(torch.uint8) neighbour = torch.cat((anns[:, :, :1], anns[:, :, :-1]), 2) edges += (anns != (0 or neighbour)).type(torch.uint8) neighbour = torch.cat((anns[:, :, 1:], anns[:, :, -1:]), 2) edges += (anns != (0 or neighbour)).type(torch.uint8) ann = anns.cpu().numpy()[3] edge = edges.cpu().numpy()[3] img = cv.cvtColor(images.cpu().numpy()[3], cv.COLOR_RGB2BGR) print(list(map(np.shape, [img, ann, edge]))) edge = cv.cvtColor(edge, cv.COLOR_GRAY2BGR) ann = cv.cvtColor(ann, cv.COLOR_GRAY2BGR) edge[np.where(edge > 0)] = 255 show = cv.hconcat([img, ann, edge]) cv.imwrite('out.jpg', show)
def __init__( self, video_list: Path, shuffle: bool, device_id: int, batch_size: int, num_workers: int, clip_length_in_frames: int, initial_prefetch_size: int, seed: int, ): self.pipe = DaliVideoPipeline( device_id=device_id, shuffle=shuffle, file_list=video_list, batch_size=batch_size, num_threads=num_workers, initial_prefetch_size=initial_prefetch_size, clip_length_in_frames=clip_length_in_frames, seed=seed, ) self.pipe.build() self.loader = DALIGenericIterator( pipelines=[self.pipe], output_map=["data", "label", "frame_idx", "start_time"], size=self.pipe.epoch_size("Reader"), )
def __init__(self, dali_pipelines, transcripts, symbols, batch_size, shard_size, train_iterator: bool): self.transcripts = transcripts self.symbols = symbols self.batch_size = batch_size from nvidia.dali.plugin.pytorch import DALIGenericIterator from nvidia.dali.plugin.base_iterator import LastBatchPolicy # in train pipeline shard_size is set to divisable by batch_size, so PARTIAL policy is safe self.dali_it = DALIGenericIterator( dali_pipelines, ["audio", "label", "audio_shape"], size=shard_size, dynamic_shape=True, auto_reset=True, last_batch_padded=True, last_batch_policy=LastBatchPolicy.PARTIAL)
def file_root_dali_iterator(batch_size, file_root, num_workers, do_shuffle, the_seed, iterator_size, reset, device, num_frames, channels): pipe = VideoPipeFileRoot(batch_size=batch_size, file_root=file_root, shuffle=do_shuffle, initial_fill=batch_size, num_threads=num_workers, seed=the_seed, device_id=device, sequence_length=num_frames, channels=channels) pipe.build() if iterator_size == 'all': it_size = pipe.epoch_size("Reader") else: it_size = iterator_size dali_iter = DALIGenericIterator([pipe], ['data', 'labels'], size=it_size, auto_reset=reset, fill_last_batch=True, last_batch_padded=False) return dali_iter
def build_dali_pipeline(args, training=True, pipe=None): # pipe is prebuilt without touching the data train_loader = DALIGenericIterator( pipelines=[pipe], output_map=['image', 'bbox', 'label'], size=pipe.epoch_size()['train_reader'] // args.N_gpu, auto_reset=True) return train_loader, pipe.epoch_size()['train_reader']
def __init__(self, params, num_workers=1, device_id=0): self.pipe = DaliPipeline(params, num_threads=num_workers, device_id=device_id) self.pipe.build() self.length = params.Nsamples self.iterator = DALIGenericIterator([self.pipe], ['inp', 'tar'], self.length, auto_reset=True)
def get_loader(flist, batch_size=512, device_id=0): pipe = ReidPipeline(flist, batch_size=batch_size, num_threads=8, device_id=device_id) pipe.build() return DALIGenericIterator(pipe, ['images', 'labels', 'camids'], size=pipe.size, auto_reset=True)
def get_train_dali_loader(args, default_boxes, local_seed): train_pipe = create_coco_pipeline(default_boxes, args, seed=local_seed) train_loader = DALIGenericIterator(train_pipe, ["images", "boxes", "labels"], reader_name="Reader", last_batch_policy=LastBatchPolicy.FILL) return train_loader
def get_train_dali_loader(args, default_boxes, local_seed): train_pipe = COCOPipeline(default_boxes, args, seed=local_seed) train_loader = DALIGenericIterator(train_pipe, ["images", "boxes", "labels"], 118287 / args.N_gpu, stop_at_epoch=False) return train_loader
def get_train_dali_loader(args, default_boxes, local_seed): train_pipe = COCOPipeline(default_boxes, args, seed=local_seed) train_loader = DALIGenericIterator(train_pipe, ["images", "boxes", "labels"], reader_name="Reader", fill_last_batch=True) return train_loader
def load_data_dali(train_data_dir, train_batch_size): # put cropped train images (use utils/data.py) with size 256x256 in train_data_dir/imgfolder/ pipe = dali.SimplePipeline(train_data_dir, batch_size=train_batch_size, num_threads=8, device_id=0) pipe.build() train_loader = DALIGenericIterator(pipe, ['data'], size=6) return train_loader
def get_iter_dali(event, batch_size, num_threads, local_rank=0, cutout=0): pip_train = HybridTrainPipe(event, batch_size=batch_size, num_threads=num_threads, device_id=local_rank, local_rank=local_rank, cutout=cutout) pip_train.build() dali_iter_train = DALIGenericIterator(pip_train, ['inputs', 'target']) # dali_iter_train= pip_train.run() return dali_iter_train
def get_train_dali_loader(args, default_boxes, local_seed): train_pipe = create_coco_pipeline(default_boxes, args, batch_size=args.batch_size, num_threads=args.num_workers, device_id=args.local_rank, seed=local_seed) train_loader = DALIGenericIterator(train_pipe, ["images", "boxes", "labels"], reader_name="Reader", last_batch_policy=LastBatchPolicy.FILL) return train_loader
def create_dali_iters(batch_size, file_list, num_workers): train_pipe = VideoPipe(batch_size=batch_size, file_list=file_list, # filenames=file_names, shuffle=False, initial_fill=2 * batch_size, num_threads=num_workers) train_pipe.build() train_dali_iter = DALIGenericIterator([train_pipe], ['data', 'labels'], train_pipe.epoch_size("Reader"), auto_reset=True) # fill_last_batch = True, last_batch_padded = False -> last batch = ``[7, 1]``, next iteration will return ``[2, 3]`` return train_dali_iter
def dataCUDA(path: list(), opt): eii = ExternalInputIterator(batch_size=opt.batch_size, root_folder=path, height=opt.height) iterator = iter(eii) pipe = ExternalSourcePipeline(data_iterator=iterator, batch_size=opt.batch_size, num_threads=opt.num_workers, device_id=0) pipe.build() # print("DALI INITIATED") data_iter = DALIGenericIterator([pipe], ['img', 'img_og'], dynamic_shape=True, size=len(path), auto_reset=False) return data_iter
class Lsun_Loader(object): def __init__(self, device_idx=1): self.batch_table = { 4: 128, 8: 128, 16: 128, 32: 64, 64: 32, 128: 16, 256: 8 } self.device_idx = device_idx #self.update(resolution_level=2) def update(self, resolution_level, batch_size): ''' :param resolution_level: resolution_level: 2-2,3-8,4-16,5-32,6-64,7-128,8-256 :return: ''' assert resolution_level >= 2 and resolution_level <= 8, "res error" self.batchsize = int(self.batch_table[pow(2, resolution_level)]) self.pipeline = TFRecordPipeline( batch_size=batch_size, size=pow(2, resolution_level), num_threads=4, device_id=self.device_idx, path= "/home/guyuchao/ssd/dataset/lsun-master/lsun_torch_tfrecord2/-r%02d.tfrecords" % resolution_level, index_path= "/home/guyuchao/ssd/dataset/lsun-master/lsun_torch_tfrecord2/-r%02d.idx" % resolution_level) self.pipeline.build() self.dali_iter = DALIGenericIterator( [self.pipeline], ["image/encoded"], self.pipeline.epoch_size("Reader"), auto_reset=True) def get_batch(self): #self.dali_iter.reset() return self.dali_iter.next()[0]["image/encoded"]
def get_loader(args, phase): file_list = args['train_video_file_list'] if phase == 'train' else None num_gpus = args['num_gpus'] batch_size_per_gpu = int(args['tem_batch_size'] / num_gpus) num_threads_per_gpu = max(int(args['data_workers'] / num_gpus), 2) pipes = [ ActivityNetVideoPipe(args, file_list, batch_size=batch_size_per_gpu, num_threads=num_threads_per_gpu, device_id=device_id) for device_id in range(1) # num_gpus) ] pipes[0].build() epoch_size = pipes[0].epoch_size("Reader") dali_iter = DALIGenericIterator(pipes, ['data', 'label'], epoch_size) return dali_iter, epoch_size
def get_dataloader(pipiter, pipline, output_map, cfg, is_train, device_id=0): if is_train: root_dir = cfg.train_datasets_bpath else: root_dir = cfg.test_datasets_bpath now_pipline = pipline(cfg, root_dir, cfg.batch_size, cfg.worker_numbers, device_id=device_id, is_train=is_train) dataloader = DALIGenericIterator(now_pipline, output_map, now_pipline.dataset.n, auto_reset=True) return dataloader
def make_loader(self): print(f" ==>> 使用训练集{self.args['tr_data_path']}训练 <<== ") train_pipe = ImagePipeline(imageset_dir=self.args['tr_data_path'], image_size=self.args["input_size"], random_shuffle=True, batch_size=self.args["batch_size"]) train_loader = DALIGenericIterator(pipelines=train_pipe, output_map=["images", "masks"], size=train_pipe.epoch_size(), auto_reset=True, fill_last_batch=False, last_batch_padded=False) if self.args['val_data_path'] != None: print(f" ==>> 使用验证集{self.args['val_data_path']}验证 <<== ") val_set = ImageFolder(self.args['val_data_path'], mode="test", in_size=self.args["input_size"], prefix=self.args['prefix']) val_loader = DataLoaderX(val_set, batch_size=self.args["batch_size"], num_workers=self.args["num_workers"], shuffle=False, drop_last=False, pin_memory=True) else: print(" ==>> 不使用验证集验证 <<== ") val_loader = None if self.args['te_data_path'] != None: print(f" ==>> 使用测试集{self.args['te_data_path']}测试 <<== ") test_set = ImageFolder(self.args['te_data_path'], mode="test", in_size=self.args["input_size"], prefix=self.args['prefix']) test_loader = DataLoaderX(test_set, batch_size=self.args["batch_size"], num_workers=self.args["num_workers"], shuffle=False, drop_last=False, pin_memory=True) else: print(f" ==>> 不使用测试集测试 <<== ") test_loader = None return train_loader, test_loader, val_loader
def get_train_dali_loader(default_boxes, root, annFile, batch_size, mean, std, local_rank, num_workers, ngpus, local_seed): train_pipe = COCOPipeline(default_boxes, root, annFile, batch_size, mean, std, local_rank, num_workers, seed=local_seed) train_loader = DALIGenericIterator(train_pipe, ["images", "boxes", "labels"], 118287 / ngpus, auto_reset=False) return train_loader