def start(self):
data_path = f"{self.config['path']['data']}{self.config['data_prefix']}{self.config['data_year']}"
csv_path = PathUtils.get_csv_data_path(self.config)
df = pd.read_csv(csv_path)
indicators = df.columns[:-3]
output_path = self.config['path']['outputs'] + "csv_to_image/"
output_dir, index_dir, img_ref_dir = FolderUtils.create_csv_to_image_output_folder(
output_path, self.config['data_prefix'], self.config['data_year'],
indicators)
FolderUtils.csv_to_image_copy_csv_files(data_path, index_dir,
img_ref_dir)
LogUtils.init_log(output_dir)
cg = CsvToImageGenerator(config=self.config,
output_dir=output_dir,
df=df,
indicators=indicators,
index_dir=index_dir,
img_ref_dir=img_ref_dir)
cg.generate_images()
def get_data_generators(self, missing_probe_file_ids):
if self._is_keras_img_model():
from data_generators.img_train_data_generator import ImgTrainDataGenerator
from data_generators.img_test_data_generator import ImgTestDataGenerator
targets_path = os.path.join(self.targets_path, "manipulation","mask")
train_gen = ImgTrainDataGenerator(
data_size=len(self.train_img_refs),
img_refs = self.train_img_refs,
patch_shape = self.patch_shape,
batch_size = self.train_batch_size,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = targets_path,
)
test_gen = ImgTestDataGenerator(
data_size=len(self.test_img_refs),
img_refs = self.test_img_refs,
patch_shape = self.patch_shape,
batch_size = self.test_batch_size,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = targets_path,
missing_probe_file_ids = missing_probe_file_ids
)
valid_gen = ImgTrainDataGenerator(
data_size=len(self.test_img_refs),
img_refs = self.test_img_refs,
batch_size = self.train_batch_size,
patch_shape = self.patch_shape,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = targets_path,
)
elif self.config['data_type'] == "image":
from data_generators.img_pixel_train_data_generator import ImgPixelTrainDataGenerator
from data_generators.img_pixel_test_data_generator import ImgPixelTestDataGenerator
train_gen = ImgPixelTrainDataGenerator(
data_size=self.train_data_size,
img_refs = self.train_img_refs,
patch_shape = self.patch_shape,
batch_size = self.train_batch_size,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = self.targets_path,
image_downscale_factor=self.image_downscale_factor
)
test_gen = ImgPixelTestDataGenerator(
data_size=self.test_data_size,
img_refs = self.test_img_refs,
patch_shape = self.patch_shape,
batch_size = self.test_batch_size,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = self.targets_path,
missing_probe_file_ids = missing_probe_file_ids,
image_downscale_factor=self.image_downscale_factor
)
valid_gen = ImgPixelTrainDataGenerator(
data_size=self.test_data_size,
img_refs = self.test_img_refs,
batch_size = self.train_batch_size,
patch_shape = self.patch_shape,
indicator_directories = self.indicator_directories,
indicators_path = self.indicators_path,
targets_path = self.targets_path,
image_downscale_factor=self.image_downscale_factor
)
elif self.config['data_type'] == 'csv':
csv_path = PathUtils.get_csv_data_path(self.config)
df = pd.read_csv(csv_path)
from data_generators.csv_pixel_test_data_generator import CsvPixelTestDataGenerator
from data_generators.csv_pixel_train_data_generator import CsvPixelTrainDataGenerator
from data_generators.csv_nn_data_generator import CsvNnDataGenerator
train_gen = CsvPixelTrainDataGenerator(
data_size=self.train_data_size,
data = df,
img_refs = self.train_img_refs,
batch_size = self.train_batch_size
)
valid_gen = CsvPixelTrainDataGenerator(
data_size=self.test_data_size,
data= df,
img_refs = self.test_img_refs,
batch_size = self.train_batch_size
)
test_gen = CsvPixelTestDataGenerator(
data_size=self.test_data_size,
data = df,
img_refs = self.test_img_refs,
batch_size = self.test_batch_size,
)
# q,w, id = test_gen.__getitem__(0)
# a,b, id = train_gen.__getitem__(0)
return train_gen, test_gen, valid_gen