def __init__(self, directory, supervised: bool = True, sequence_length: int = 11, batch_size: int = 1, num_workers: int = 1, gpu_id: int = 0, shuffle: bool = True, crop_size: tuple = (256, 256), mean: list = [0.5, 0.5, 0.5], std: list = [0.5, 0.5, 0.5], conv_mode: str = '3d', validate: bool = False, distributed: bool = False): self.pipeline = KineticsDALIPipe(directory=directory, batch_size=batch_size, supervised=supervised, sequence_length=sequence_length, num_workers=num_workers, gpu_id=gpu_id, crop_size=crop_size, mean=mean, std=std, conv_mode=conv_mode, validate=validate) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size("Reader") names = ['images', 'labels'] if supervised else ['images'] self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline, names, self.epoch_size, auto_reset=True)
def __init__(self, batch_size, file_root, sequence_length, crop_size, epoch_size=-1, random_shuffle=True, temp_stride=-1): # Builds list of sequence filenames container_files = os.listdir(file_root) container_files = [file_root + '/' + f for f in container_files] # Define and build pipeline self.pipeline = VideoReaderPipeline(batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=0, files=container_files, crop_size=crop_size, random_shuffle=random_shuffle, step=temp_stride) self.pipeline.build() # Define size of epoch if epoch_size <= 0: self.epoch_size = self.pipeline.epoch_size("Reader") else: self.epoch_size = epoch_size self.dali_iterator = pytorch.DALIGenericIterator(pipelines=self.pipeline, output_map=["data"], size=self.epoch_size, auto_reset=True)
def __init__(self, batch_size, file_root, sequence_length): self.pipeline = SimpleSequencePipeline(batch_size, file_root, sequence_length) self.pipeline.build() self.epoch_size = list(self.pipeline.epoch_size().values())[0] self.dali_iterator = pytorch.DALIGenericIterator( self.pipeline, ["data"], self.epoch_size)
def __init__(self, file_list, batch_size, sequence_length, crop_size, epoch_size=-1, random_shuffle=True, step=-1, stride=1, device_id=0): # Define and build pipeline self.pipeline = VideoReaderPipeline(file_list=file_list, batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=device_id, crop_size=crop_size, random_shuffle=random_shuffle, step=step, stride=stride) self.pipeline.build() # Define size of epoch if epoch_size <= 0: self.epoch_size = self.pipeline.epoch_size("Reader") else: self.epoch_size = epoch_size # Define Pytorch tensor iterator self.dali_iterator = pytorch.DALIGenericIterator( pipelines=self.pipeline, output_map=["data", "labels"], size=self.epoch_size, auto_reset=True)
def dali_ilsvrc_loader(mx_record_dir, num_gpus, batch_size, crop_size=224, resize=256, num_threads_per_gpu=1): pipes = [HybridTrainPipe(batch_size=int(batch_size/num_gpus), num_threads=num_threads_per_gpu, device_id=device_id,\ record_dir=mx_record_dir, crop=crop_size, num_gpus=num_gpus, dali_cpu=False) for device_id in range(num_gpus)] pipes[0].build() train_loader = plugin_pytorch.DALIGenericIterator( pipes, ["data", "label"], size=int(pipes[0].epoch_size("Reader"))) pipes = [HybridValPipe(batch_size=int(100/num_gpus), num_threads=num_threads_per_gpu, device_id=device_id,\ record_dir=mx_record_dir, crop=crop_size, size=resize, num_gpus=num_gpus, dali_cpu=False) for device_id in range(num_gpus)] pipes[0].build() val_loader = plugin_pytorch.DALIGenericIterator( pipes, ["data", "label"], size=int(pipes[0].epoch_size("Reader"))) return train_loader, val_loader
def __init__( self, batch_size, image0_names, image1_names, flow_names, valBool, num_threads, device_id, ): self.pipeline = create_image_pipeline( batch_size, num_threads, device_id, image0_names, image1_names, flow_names, valBool, ) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size( "Reader") / batch_size output_names = ["images", "flow"] if valBool: self.dali_iterator = pytorch.DALIGenericIterator( self.pipeline, output_names, reader_name="Reader", last_batch_policy=pytorch.LastBatchPolicy.PARTIAL, auto_reset=True, ) else: self.dali_iterator = pytorch.DALIGenericIterator( self.pipeline, output_names, reader_name="Reader", last_batch_policy=pytorch.LastBatchPolicy.PARTIAL, auto_reset=True, )
def _construct_loader(self): """ Construct the video loader. """ data_file = 'val' if self.mode == 'test' else self.mode path_to_file = os.path.join(self.file_root, "{}.csv".format(data_file)) assert os.path.exists(path_to_file), "{} dir not found".format( path_to_file) self._classes = [] with open(os.path.join(self.file_root, 'moments_categories.txt')) as f: self._classes = f.readlines() self._classes = [c.split(',')[0] for c in self._classes] self._path_to_videos = [] self._labels = [] self._spatial_temporal_idx = [] self._video_meta = {} self._num_clips = 1 print('loading files') with open(path_to_file, "r") as f: for clip_idx, path_label in enumerate(f.readlines()): if clip_idx < 800: # assert len(path_label.split()) == 4 path, label = path_label.split(',')[:2] for idx in range(self._num_clips): self._path_to_videos.append( os.path.join(self.file_root, data_file, path)) self._labels.append(self._classes.index(label)) self._spatial_temporal_idx.append(idx) self._video_meta[clip_idx * self._num_clips + idx] = {} print('creating pipeline') self.pipeline = VideoReaderPipeline( batch_size=self.batch_size, sequence_length=self.sequence_length, num_threads=8, device_id=0, files=self._path_to_videos, crop_size=self.crop_size, # stride=8 ) print('building pipeline') self.pipeline.build() self.epoch_size = self.pipeline.epoch_size("Reader") print('building iterator') self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline, ["data"], self.epoch_size, auto_reset=True)
def __init__(self, batch_size, file_root, sequence_length, crop_size): container_files = os.listdir(file_root) container_files = [file_root + '/' + f for f in container_files] self.pipeline = VideoReaderPipeline(batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=0, files=container_files, crop_size=crop_size) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size("Reader") self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline, ["data"], self.epoch_size, auto_reset=True)
def __init__(self, batch_size, file_list, uid2label, sequence_length, crop_size): self.pipeline = VideoReaderPipeline(batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=0, file_list=file_list, crop_size=crop_size) self.pipeline.build() self.uid2label = uid2label self.epoch_size = self.pipeline.epoch_size("Reader") self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline, ["data", "uid", "frame_num", "crop_pos_x", "crop_pos_y", "is_flipped"], self.epoch_size, auto_reset=True)
def __init__(self, batch_size, file_root, sequence_length, crop_size): container_files = os.listdir(file_root) container_files = [file_root + '/' + f for f in container_files] self.pipeline = create_video_reader_pipeline( batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=0, files=container_files, crop_size=crop_size) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size("Reader") self.dali_iterator = pytorch.DALIGenericIterator( self.pipeline, ["data"], reader_name="Reader", last_batch_policy=pytorch.LastBatchPolicy.PARTIAL, auto_reset=True)
def __init__(self, batch_size, file_root, sequence_length, crop_size, transforms=None): # container_files = [file_root + '/' + f for f in os.listdir(file_root)] self.pipeline = VideoReaderPipeline( batch_size=batch_size, sequence_length=sequence_length, num_threads=2, device_id=0, file_root=file_root, crop_size=crop_size, transforms=transforms ) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size('Reader') self.dali_iterator = pytorch.DALIGenericIterator( self.pipeline, ["data", "label"], self.epoch_size, auto_reset=True )
def __init__(self, batch_size, file_root, gt_file_root, sequence_length, \ crop_size, epoch_size=-1, random_shuffle=False, temp_stride=-1, device_id=0): # Builds list of sequence filenames container_files = os.listdir(file_root) container_files.sort() # F**K os.listdir() container_files = [file_root + '/' + f for f in container_files] gt_container_files = os.listdir(gt_file_root) gt_container_files.sort() gt_container_files = [ gt_file_root + '/' + f for f in gt_container_files ] # print(container_files) # print(gt_container_files) # Define and build pipeline self.pipeline = VideoReaderPipeline(batch_size=batch_size, \ sequence_length=sequence_length, \ num_threads=1, \ device_id=device_id, \ files=container_files, \ gt_files=gt_container_files, \ crop_size=crop_size, \ random_shuffle=random_shuffle,\ step=temp_stride) self.pipeline.build() if epoch_size <= 0: self.epoch_size = self.pipeline.epoch_size("Reader") else: self.epoch_size = epoch_size self.dali_iterator = pytorch.DALIGenericIterator(pipelines=self.pipeline, \ output_map=["noisy","gt"], size=self.epoch_size, \ auto_reset=True) print("DALI initialize successfully")
def __init__(self, idx, video_file_name, ts_file_name, batch_size, sequence_length): self.idx = idx self.video_file_name = video_file_name self.init_time_stamps(ts_file_name, video_file_name) self.pipeline = VideoReaderPipeline(batch_size=batch_size, sequence_length=sequence_length, num_threads=1, device_id=0, files=(video_file_name)) self.pipeline.build() self.epoch_size = self.pipeline.epoch_size("Reader") drop_tail = len(self.ts) - self.epoch_size * sequence_length # eliminate time stamps that will be dropped due to the DALI # layer deliviring only complete sequences full_length = len(self.ts) if drop_tail > 0: self.ts = self.ts[:-drop_tail] print('video %s has %7d frames, trimmed to %7d' % (self.video_file_name, full_length, len(self.ts))) self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline, ["data"], self.epoch_size, auto_reset=True)
def test_pytorch_plugin_cpu(): pipe = Pipeline(batch_size=batch_size, num_threads=3, device_id=None) outs = fn.external_source(source=get_data, layout="HWC") pipe.set_outputs(outs) pii = pytorch.DALIGenericIterator([pipe], ["data"])