def test_dataloader(self): """Summary Returns: TYPE: Description """ ds_test = MultiLabelDataset(folder=self.hparams.data_path, is_train='test', fname='test.csv', types=self.hparams.types, pathology=self.hparams.pathology, resize=int(self.hparams.shape)) ds_test.reset_state() ag_test = [ imgaug.Albumentations( AB.SmallestMaxSize(self.hparams.shape, p=1.0)), iimgaug.ColorSpace(mode=cv2.COLOR_GRAY2RGB), imgaug.Albumentations(AB.CLAHE(p=1)), imgaug.ToFloat32(), ] ds_test = AugmentImageComponent(ds_test, ag_test, 0) ds_test = BatchData(ds_test, self.hparams.batch, remainder=True) # ds_test = MultiProcessRunner(ds_test, num_proc=4, num_prefetch=16) ds_test = PrintData(ds_test) ds_test = MapData(ds_test, lambda dp: [torch.tensor(np.transpose(dp[0], (0, 3, 1, 2))), torch.tensor(dp[1]).float()]) return ds_test
def val_dataloader(self): """Summary Returns: TYPE: Description """ ds_valid = MultiLabelDataset( folder=self.hparams.data, is_train='valid', fname='covid_test_v5.csv', types=self.hparams.types, pathology=self.hparams.pathology, resize=int(self.hparams.shape), ) ds_valid.reset_state() ag_valid = [ imgaug.Resize(self.hparams.shape, interp=cv2.INTER_AREA), imgaug.ToFloat32(), ] ds_valid = AugmentImageComponent(ds_valid, ag_valid, 0) ds_valid = BatchData(ds_valid, self.hparams.batch, remainder=True) ds_valid = MultiProcessRunner(ds_valid, num_proc=4, num_prefetch=16) ds_valid = PrintData(ds_valid) ds_valid = MapData( ds_valid, lambda dp: [ torch.tensor(np.transpose(dp[0], (0, 3, 1, 2))), torch.tensor(dp[1]).float() ]) return ds_valid
def test_dataloader(self): """Summary Returns: TYPE: Description """ ds_test = CustomDataSet(folder=self.hparams.data, train_or_valid='test', size=np.inf, hparams=self.hparams) ds_test.reset_state() ag_test = [ imgaug.Resize(self.hparams.shape, interp=cv2.INTER_NEAREST), imgaug.ToFloat32(), ] # ds_test = AugmentImageComponent(ds_test, [imgaug.Albumentations(AB.CLAHE(tile_grid_size=(32, 32), always_apply=True, p=1)),], 0) ds_test = AugmentImageComponents(ds_test, ag_test, [0, 1]) ds_test = BatchData(ds_test, self.hparams.batch, remainder=True) ds_test = MultiProcessRunner(ds_test, num_proc=4, num_prefetch=16) ds_test = PrintData(ds_test) ds_test = MapData( ds_test, lambda dp: [ torch.tensor(dp[0][:, np.newaxis, :, :]).float(), torch.tensor(dp[1][:, np.newaxis, :, :]).float(), ]) return ds_test
def test_dataloader(self): ds_test = MultiLabelDataset(folder=self.hparams.data, is_train='valid', fname='covid_test_v5.csv', types=self.hparams.types, pathology=self.hparams.pathology, resize=int(self.hparams.shape), fold_idx=None, n_folds=1) ds_test.reset_state() ag_test = [ imgaug.Resize(self.hparams.shape, interp=cv2.INTER_AREA), imgaug.ToFloat32(), ] ds_test = AugmentImageComponent(ds_test, ag_test, 0) ds_test = BatchData(ds_test, self.hparams.batch, remainder=True) ds_test = MultiProcessRunner(ds_test, num_proc=4, num_prefetch=16) ds_test = PrintData(ds_test) ds_test = MapData( ds_test, lambda dp: [ torch.tensor(np.transpose(dp[0], (0, 3, 1, 2))), torch.tensor(dp[1]).float() ]) return ds_test
def train_dataloader(self): ds_train = MultiLabelDataset(folder=self.hparams.data, is_train='train', fname='covid_train_v5.csv', types=self.hparams.types, pathology=self.hparams.pathology, resize=int(self.hparams.shape), balancing=None) ds_train.reset_state() ag_train = [ # imgaug.Albumentations( # AB.SmallestMaxSize(self.hparams.shape, p=1.0)), imgaug.ColorSpace(mode=cv2.COLOR_GRAY2RGB), # imgaug.Affine(shear=10), imgaug.RandomChooseAug([ imgaug.Albumentations(AB.Blur(blur_limit=4, p=0.25)), imgaug.Albumentations(AB.MotionBlur(blur_limit=4, p=0.25)), imgaug.Albumentations(AB.MedianBlur(blur_limit=4, p=0.25)), ]), imgaug.Albumentations(AB.CLAHE(tile_grid_size=(32, 32), p=0.5)), imgaug.RandomOrderAug([ imgaug.Affine(shear=10, border=cv2.BORDER_CONSTANT, interp=cv2.INTER_AREA), imgaug.Affine(translate_frac=(0.01, 0.02), border=cv2.BORDER_CONSTANT, interp=cv2.INTER_AREA), imgaug.Affine(scale=(0.5, 1.0), border=cv2.BORDER_CONSTANT, interp=cv2.INTER_AREA), ]), imgaug.RotationAndCropValid(max_deg=10, interp=cv2.INTER_AREA), imgaug.GoogleNetRandomCropAndResize( crop_area_fraction=(0.8, 1.0), aspect_ratio_range=(0.8, 1.2), interp=cv2.INTER_AREA, target_shape=self.hparams.shape), imgaug.ColorSpace(mode=cv2.COLOR_RGB2GRAY), imgaug.ToFloat32(), ] ds_train = AugmentImageComponent(ds_train, ag_train, 0) # Label smoothing ag_label = [ imgaug.BrightnessScale((0.8, 1.2), clip=False), ] # ds_train = AugmentImageComponent(ds_train, ag_label, 1) ds_train = BatchData(ds_train, self.hparams.batch, remainder=True) if self.hparams.debug: ds_train = FixedSizeData(ds_train, 2) ds_train = MultiProcessRunner(ds_train, num_proc=4, num_prefetch=16) ds_train = PrintData(ds_train) ds_train = MapData( ds_train, lambda dp: [ torch.tensor(np.transpose(dp[0], (0, 3, 1, 2))), torch.tensor(dp[1]).float() ]) return ds_train
def predict_all(self, data_path: str, isSrgb: bool, isLdr: bool): t0 = time.time() df = InferenceDataflow(data_path, self.step, isSrgb, isLdr) df = PrintData(df, max_list=7, max_depth=7) df.reset_state() # Needed to setup dataflow for dp in tqdm(df): path, dp = dp self.predict(dp, path) t1 = time.time() print("Prediction finished in: {}".format(t1 - t0))
def render_all(self, data_path: str): t0 = time.time() df = InferenceDataflow(data_path, self.step) df = PrintData(df) df.reset_state() # Needed to setup dataflow for dp in tqdm(df): path, dp = dp self.render(dp, path) t1 = time.time() print("Rendering finished in: {}".format(t1 - t0))
def val_dataloader(self): """Summary Returns: TYPE: Description """ ds_valid = CustomDataSet(folder=self.hparams.data, train_or_valid='valid', size=np.inf, hparams=self.hparams) ds_valid.reset_state() ag_valid = [ imgaug.Resize(self.hparams.shape, interp=cv2.INTER_NEAREST), imgaug.ToFloat32(), ] ds_valid = AugmentImageComponent(ds_valid, [ imgaug.Albumentations(AB.CLAHE(p=1)), ], 0) if self.hparams.types == 6: ds_valid = AugmentImageComponents(ds_valid, ag_valid, [0, 1, 2, 3, 4, 5, 6]) elif self.hparams.types == 1: ds_valid = AugmentImageComponents(ds_valid, ag_valid, [0, 1]) ds_valid = BatchData(ds_valid, self.hparams.batch, remainder=True) ds_valid = MultiProcessRunner(ds_valid, num_proc=4, num_prefetch=16) ds_valid = PrintData(ds_valid) if self.hparams.types == 6: ds_valid = MapData( ds_valid, lambda dp: [ torch.tensor(dp[0][:, np.newaxis, :, :]).float(), torch.tensor(dp[1][:, np.newaxis, :, :]).float(), torch.tensor(dp[2][:, np.newaxis, :, :]).float(), torch.tensor(dp[3][:, np.newaxis, :, :]).float(), torch.tensor(dp[4][:, np.newaxis, :, :]).float(), torch.tensor(dp[5][:, np.newaxis, :, :]).float(), torch.tensor(dp[6][:, np.newaxis, :, :]).float(), ]) elif self.hparams.types == 1: ds_valid = MapData( ds_valid, lambda dp: [ torch.tensor(dp[0][:, np.newaxis, :, :]).float(), torch.tensor(dp[1][:, np.newaxis, :, :]).float(), ]) return ds_valid
add_gt=False) num_imgs = len(imgs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(imgs[img_range[0]:img_range[1]], ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds if __name__ == '__main__': import os from tensorpack.dataflow import PrintData cfg.DATA.BASEDIR = os.path.expanduser('~/data/coco') ds = get_train_dataflow() ds = PrintData(ds, 100) TestDataSpeed(ds, 50000).start() ds.reset_state() for k in ds.get_data(): pass
logger.info("Found {} images for inference.".format(len(roidbs))) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]: img_range[1]], ["file_name", "image_id"]) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds if __name__ == "__main__": import os from tensorpack.dataflow import PrintData cfg.DATA.BASEDIR = os.path.expanduser("~/data/coco") ds = get_train_dataflow() ds = PrintData(ds, 100) TestDataSpeed(ds, 50000).start() ds.reset_state() for k in ds: pass
img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]:img_range[1]], ["file_name", "image_id"]) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds if __name__ == "__main__": import os from tensorpack.dataflow import PrintData from config import finalize_configs register_coco(os.path.expanduser("~/data/coco")) finalize_configs() ds = get_train_dataflow() ds = PrintData(ds, 10) TestDataSpeed(ds, 50000).start() for _ in ds: pass
return ret ds = MultiProcessMapData(ds, 3, preprocess) return ds def get_eval_dataflow(): imgs = COCODetection.load_many(config.BASEDIR, config.VAL_DATASET, add_gt=False) # no filter for training ds = DataFromListOfDict(imgs, ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) ds = PrefetchDataZMQ(ds, 1) return ds if __name__ == '__main__': config.BASEDIR = '/private/home/yuxinwu/data/coco' config.TRAIN_DATASET = ['train2014'] from tensorpack.dataflow import PrintData ds = get_train_dataflow(add_mask=config.MODE_MASK) ds = PrintData(ds, 100) TestDataSpeed(ds, 50000).start() ds.reset_state() for k in ds.get_data(): pass
def train_dataloader(self): """Summary Returns: TYPE: Description """ ds_train = CustomDataSet(folder=self.hparams.data, train_or_valid='train', size=np.inf, hparams=self.hparams) ds_train.reset_state() ag_train = [ imgaug.Affine(shear=10, interp=cv2.INTER_NEAREST), imgaug.Affine(translate_frac=(0.01, 0.02), interp=cv2.INTER_NEAREST), imgaug.Affine(scale=(0.25, 1.0), interp=cv2.INTER_NEAREST), imgaug.RotationAndCropValid(max_deg=10, interp=cv2.INTER_NEAREST), imgaug.GoogleNetRandomCropAndResize( crop_area_fraction=(0.8, 1.0), aspect_ratio_range=(0.8, 1.2), interp=cv2.INTER_NEAREST, target_shape=self.hparams.shape), imgaug.Resize(self.hparams.shape, interp=cv2.INTER_NEAREST), imgaug.Flip(horiz=True, vert=False, prob=0.5), imgaug.Flip(horiz=False, vert=True, prob=0.5), imgaug.Transpose(prob=0.5), imgaug.Albumentations(AB.RandomRotate90(p=1)), imgaug.ToFloat32(), ] ds_train = AugmentImageComponent( ds_train, [ # imgaug.Float32(), # imgaug.RandomChooseAug([ # imgaug.Albumentations(AB.IAAAdditiveGaussianNoise(p=0.25)), # imgaug.Albumentations(AB.GaussNoise(p=0.25)), # ]), # imgaug.ToUint8(), imgaug.RandomChooseAug([ imgaug.Albumentations(AB.Blur(blur_limit=4, p=0.25)), imgaug.Albumentations(AB.MotionBlur(blur_limit=4, p=0.25)), imgaug.Albumentations(AB.MedianBlur(blur_limit=4, p=0.25)), ]), imgaug.RandomChooseAug([ # imgaug.Albumentations(AB.IAASharpen(p=0.5)), # imgaug.Albumentations(AB.IAAEmboss(p=0.5)), imgaug.Albumentations(AB.RandomBrightnessContrast(p=0.5)), ]), imgaug.ToUint8(), imgaug.Albumentations(AB.CLAHE(tile_grid_size=(32, 32), p=0.5)), ], 0) ds_train = AugmentImageComponents(ds_train, ag_train, [0, 1]) ds_train = BatchData(ds_train, self.hparams.batch, remainder=True) if self.hparams.debug: ds_train = FixedSizeData(ds_train, 2) ds_train = MultiProcessRunner(ds_train, num_proc=4, num_prefetch=16) ds_train = PrintData(ds_train) ds_train = MapData( ds_train, lambda dp: [ torch.tensor(dp[0][:, np.newaxis, :, :]).float(), torch.tensor(dp[1][:, np.newaxis, :, :]).float(), ]) return ds_train
# Mark Boss, Varun Jampani, Kihwan Kim, Hendrik P. A. Lensch, Jan Kautz # ----------------------------------------------------------------------- from dataflow.dataflow import Dataflows, get_data if __name__ == "__main__": import argparse from tensorpack.dataflow import TestDataSpeed, PrintData import tqdm parser = argparse.ArgumentParser() parser.add_argument("--folder", required=True) parser.add_argument("--batch_size", type=int, default=8) parser.add_argument("--stage", type=int, default=0, choices=[0, 1, 2, 3]) args = parser.parse_args() print(args) if args.stage == 0: df = Dataflows.SHAPE elif args.stage == 1: df = Dataflows.ILLUMINATION elif args.stage == 2: df = Dataflows.BRDF elif args.stage == 3: df = Dataflows.JOINT ds = get_data(df, args.folder, args.batch_size) ds = PrintData(ds) TestDataSpeed(ds, 1000).start()
Args: shard, num_shards: to get subset of evaluation data """ roidbs = COCODetection.load_many(cfg.DATA.BASEDIR, cfg.DATA.VAL, add_gt=False) num_imgs = len(roidbs) img_per_shard = num_imgs // num_shards img_range = (shard * img_per_shard, (shard + 1) * img_per_shard if shard + 1 < num_shards else num_imgs) # no filter for training ds = DataFromListOfDict(roidbs[img_range[0]: img_range[1]], ['file_name', 'id']) def f(fname): im = cv2.imread(fname, cv2.IMREAD_COLOR) assert im is not None, fname return im ds = MapDataComponent(ds, f, 0) # Evaluation itself may be multi-threaded, therefore don't add prefetch here. return ds if __name__ == '__main__': import os from tensorpack.dataflow import PrintData cfg.DATA.BASEDIR = os.path.expanduser('~/data/coco') ds = get_train_dataflow() ds = PrintData(ds, 100) TestDataSpeed(ds, 50000).start() ds.reset_state() for k in ds: pass