def start(self): data_path = f"{self.config['path']['data']}{self.config['data_prefix']}{self.config['data_year']}" csv_path = PathUtils.get_csv_data_path(self.config) df = pd.read_csv(csv_path) indicators = df.columns[:-3] output_path = self.config['path']['outputs'] + "csv_to_image/" output_dir, index_dir, img_ref_dir = FolderUtils.create_csv_to_image_output_folder( output_path, self.config['data_prefix'], self.config['data_year'], indicators) FolderUtils.csv_to_image_copy_csv_files(data_path, index_dir, img_ref_dir) LogUtils.init_log(output_dir) cg = CsvToImageGenerator(config=self.config, output_dir=output_dir, df=df, indicators=indicators, index_dir=index_dir, img_ref_dir=img_ref_dir) cg.generate_images()
def get_data_generators(self, missing_probe_file_ids): if self._is_keras_img_model(): from data_generators.img_train_data_generator import ImgTrainDataGenerator from data_generators.img_test_data_generator import ImgTestDataGenerator targets_path = os.path.join(self.targets_path, "manipulation","mask") train_gen = ImgTrainDataGenerator( data_size=len(self.train_img_refs), img_refs = self.train_img_refs, patch_shape = self.patch_shape, batch_size = self.train_batch_size, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = targets_path, ) test_gen = ImgTestDataGenerator( data_size=len(self.test_img_refs), img_refs = self.test_img_refs, patch_shape = self.patch_shape, batch_size = self.test_batch_size, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = targets_path, missing_probe_file_ids = missing_probe_file_ids ) valid_gen = ImgTrainDataGenerator( data_size=len(self.test_img_refs), img_refs = self.test_img_refs, batch_size = self.train_batch_size, patch_shape = self.patch_shape, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = targets_path, ) elif self.config['data_type'] == "image": from data_generators.img_pixel_train_data_generator import ImgPixelTrainDataGenerator from data_generators.img_pixel_test_data_generator import ImgPixelTestDataGenerator train_gen = ImgPixelTrainDataGenerator( data_size=self.train_data_size, img_refs = self.train_img_refs, patch_shape = self.patch_shape, batch_size = self.train_batch_size, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = self.targets_path, image_downscale_factor=self.image_downscale_factor ) test_gen = ImgPixelTestDataGenerator( data_size=self.test_data_size, img_refs = self.test_img_refs, patch_shape = self.patch_shape, batch_size = self.test_batch_size, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = self.targets_path, missing_probe_file_ids = missing_probe_file_ids, image_downscale_factor=self.image_downscale_factor ) valid_gen = ImgPixelTrainDataGenerator( data_size=self.test_data_size, img_refs = self.test_img_refs, batch_size = self.train_batch_size, patch_shape = self.patch_shape, indicator_directories = self.indicator_directories, indicators_path = self.indicators_path, targets_path = self.targets_path, image_downscale_factor=self.image_downscale_factor ) elif self.config['data_type'] == 'csv': csv_path = PathUtils.get_csv_data_path(self.config) df = pd.read_csv(csv_path) from data_generators.csv_pixel_test_data_generator import CsvPixelTestDataGenerator from data_generators.csv_pixel_train_data_generator import CsvPixelTrainDataGenerator from data_generators.csv_nn_data_generator import CsvNnDataGenerator train_gen = CsvPixelTrainDataGenerator( data_size=self.train_data_size, data = df, img_refs = self.train_img_refs, batch_size = self.train_batch_size ) valid_gen = CsvPixelTrainDataGenerator( data_size=self.test_data_size, data= df, img_refs = self.test_img_refs, batch_size = self.train_batch_size ) test_gen = CsvPixelTestDataGenerator( data_size=self.test_data_size, data = df, img_refs = self.test_img_refs, batch_size = self.test_batch_size, ) # q,w, id = test_gen.__getitem__(0) # a,b, id = train_gen.__getitem__(0) return train_gen, test_gen, valid_gen