def test_zips_no_continent_data_correctly(): opt = NullOptions() opt.dataroot = os.path.expanduser('~/data/new_geo_data') opt.phase = '' opt.resize_or_crop = 'resize_and_crop' opt.loadSize = 256 opt.fineSize = 256 opt.which_direction = 'AtoB' opt.input_nc = 1 opt.output_nc = 1 opt.no_flip = True, opt.div_threshold = 1000 opt.inpaint_single_class = False opt.continent_data = True geo = GeoDataset() geo.initialize(opt) # print(geo.A_paths) for paths in geo.A_paths: def get_subfolder(path): return os.path.basename(os.path.dirname(path)) subfolders = [get_subfolder(path) for path in paths] s_nos = [get_series_number(path) for path in paths] assert (all( [subfolder == subfolders[0] for subfolder in subfolders[1:]])) assert (all([s_no == s_nos[0] for s_no in s_nos[1:]]))
def new_dataset(): opt = NullOptions() opt.dataroot = 'test_data/with_continents' opt.phase = '' opt.resize_or_crop = 'resize_and_crop' opt.loadSize = 256 opt.fineSize = 256 opt.which_direction = 'AtoB' opt.input_nc = 1 opt.output_nc = 1 opt.no_flip = True, opt.div_threshold = 1000 opt.inpaint_single_class = False opt.continent_data = True geo = GeoDataset() geo.initialize(opt) return geo
def dataset(pytestconfig): opt = NullOptions() opt.dataroot = os.path.expanduser(pytestconfig.option.dataroot) opt.phase = 'test' # opt.resize_or_crop = 'resize_and_crop' opt.loadSize = 256 opt.fineSize = 256 opt.which_direction = 'AtoB' opt.input_nc = 1 opt.output_nc = 1 opt.no_flip = True opt.div_threshold = 1000 opt.inpaint_single_class = False opt.continent_data = False geo = GeoDataset() geo.initialize(opt) return geo
def test_default_continent_map_is_blank(): opt = NullOptions() opt.dataroot = 'test_data/no_continents' opt.phase = '' opt.resize_or_crop = 'resize_and_crop' opt.loadSize = 256 opt.fineSize = 256 opt.which_direction = 'AtoB' opt.input_nc = 1 opt.output_nc = 1 opt.no_flip = True, opt.div_threshold = 1000 opt.inpaint_single_class = False opt.continent_data = True geo = GeoDataset() geo.initialize(opt) data = geo[0] assert (torch.sum(data['continents']) == 0)
def CreateDataset(opt): dataset = None # Data stored as one image concatenated along horizontal axis if opt.dataset_mode == 'aligned': from data.aligned_dataset import AlignedDataset dataset = AlignedDataset() # Data stored in different directories elif opt.dataset_mode == 'unaligned': from data.unaligned_dataset import UnalignedDataset dataset = UnalignedDataset() elif opt.dataset_mode == 'geo': from data.geo_dataset import GeoDataset dataset = GeoDataset() elif opt.dataset_mode == 'single': from data.single_dataset import SingleDataset dataset = SingleDataset() else: raise ValueError("Dataset [%s] not recognized." % opt.dataset_mode) print("dataset [%s] was created" % (dataset.name())) dataset.initialize(opt) return dataset
# tensorboard 出力 board_train = SummaryWriter( log_dir=os.path.join(args.tensorboard_dir, args.exper_name)) board_valid = SummaryWriter( log_dir=os.path.join(args.tensorboard_dir, args.exper_name + "_valid")) board_eval = SummaryWriter( log_dir=os.path.join(args.tensorboard_dir, args.exper_name + "_eval")) #================================ # データセットの読み込み #================================ # 学習用データセットとテスト用データセットの設定 ds_train = GeoDataset(args, args.dataset_train_dir, image_height=args.image_height, image_width=args.image_width, data_augument=args.data_augument, geometric_model=args.geometric_model, debug=args.debug) index = np.arange(len(ds_train)) train_index, valid_index = train_test_split(index, test_size=args.val_rate, random_state=args.seed) if (args.debug): print("train_index.shape : ", train_index.shape) print("valid_index.shape : ", valid_index.shape) print("train_index[0:10] : ", train_index[0:10]) print("valid_index[0:10] : ", valid_index[0:10]) dloader_train = torch.utils.data.DataLoader(Subset(ds_train, train_index),
def temp_dataset(dataset, folder_nums=[1, 2, 3, 4]): dataroot = dataset.opt.dataroot # Create a temporary directory to test temp_data_parent = tempfile.mkdtemp(dir='/tmp') folder_labels = ['{:02}'.format(num) for num in folder_nums] temp_data_dir_1 = os.path.join(temp_data_parent, folder_labels[0]) temp_data_dir_2 = os.path.join(temp_data_parent, folder_labels[1]) temp_data_dir_3 = os.path.join(temp_data_parent, folder_labels[2]) temp_data_dir_4 = os.path.join(temp_data_parent, folder_labels[3]) os.mkdir(temp_data_dir_1) os.mkdir(temp_data_dir_2) os.mkdir(temp_data_dir_3) os.mkdir(temp_data_dir_4) [ shutil.copy(file, temp_data_dir_1) for file in glob.glob(dataroot + '/test/serie100001_project_*.dat') ] [ shutil.copy(file, temp_data_dir_2) for file in glob.glob(dataroot + '/test/serie100002_project_*.dat') ] [ shutil.copy(file, temp_data_dir_3) for file in glob.glob(dataroot + '/test/serie100003_project_*.dat') ] [ shutil.copy(file, temp_data_dir_3) for file in glob.glob(dataroot + '/test/serie100004_project_*.dat') ] [ shutil.copy(file, temp_data_dir_4) for file in glob.glob(dataroot + '/test/serie100004_project_*.dat') ] [ shutil.copy(file, temp_data_parent) for file in glob.glob(dataroot + '/test/serie100004_project_*.dat') ] for folder in [ temp_data_parent, temp_data_dir_1, temp_data_dir_2, temp_data_dir_3, temp_data_dir_4 ]: for tag in ['DIV', 'Vx', 'Vy']: with open(os.path.join(folder, tag + '_norm.dat'), 'w') as file: file.write('-10000 10000') # Check they're in the target directory glob.glob(os.path.join(temp_data_parent, '*.dat'))[0] glob.glob(os.path.join(temp_data_dir_1, '*.dat'))[0] glob.glob(os.path.join(temp_data_dir_2, '*.dat'))[0] glob.glob(os.path.join(temp_data_dir_3, '*.dat'))[0] glob.glob(os.path.join(temp_data_dir_4, '*.dat'))[0] # Now build a second dataset using this dummy directory opt = NullOptions() opt.dataroot = temp_data_parent opt.phase = '' opt.resize_or_crop = 'resize_and_crop' opt.loadSize = 256 opt.fineSize = 256 opt.which_direction = 'AtoB' opt.input_nc = 1 opt.output_nc = 1 opt.no_flip = True opt.div_threshold = 1000 opt.inpaint_single_class = False opt.continent_data = False geo = GeoDataset() geo.initialize(opt) div_files, vx_files, vy_files, _ = geo.get_dat_files(temp_data_parent) assert (len(div_files) == 6) assert (len(vx_files) == 6) assert (len(vy_files) == 6) geo.initialize(opt) return geo, opt