def create_generators(args, data, DeepForest_config):
""" Create generators for training and validation.
"""
#Split training and test data
train, test = preprocess.split_training(data, DeepForest_config, experiment=None)
#Write out for debug
if args.save_path:
train.to_csv(os.path.join(args.save_path,'training_dict.csv'), header=False)
#Training Generator
train_generator = H5Generator(train,
batch_size = args.batch_size,
DeepForest_config = DeepForest_config,
name = "training",
preprocess_image=image_utils.normalize
)
#Validation Generator, check that it exists
if test is not None:
validation_generator = H5Generator(test,
batch_size = args.batch_size,
DeepForest_config = DeepForest_config,
name = "training",
preprocess_image=image_utils.normalize
)
else:
validation_generator = None
return train_generator, validation_generator
def create_h5_generators(data, DeepForest_config):
""" Create generators for training and validation.
"""
#Split training and test data
train, test = split_training(data, DeepForest_config, experiment=None)
#Write out for debug
if DeepForest_config["save_image_path"]:
train.to_csv(os.path.join(DeepForest_config["save_image_path"],
'training_dict.csv'),
header=False)
if DeepForest_config["spatial_filter"]:
train = spatial_filter(train, DeepForest_config)
#Training Generator
train_generator = H5Generator(train,
batch_size=DeepForest_config["batch_size"],
DeepForest_config=DeepForest_config,
name="training",
preprocess_image=image_utils.preprocess)
#Validation Generator, check that it exists
if test is not None:
validation_generator = H5Generator(
test,
batch_size=DeepForest_config["batch_size"],
DeepForest_config=DeepForest_config,
name="training",
preprocess_image=image_utils.preprocess)
else:
validation_generator = None
return train_generator, validation_generator
def test_split_training(DeepForest_config):
data = generators.load_retraining_data(DeepForest_config)
train, test = preprocess.split_training(
data, DeepForest_config=DeepForest_config, experiment=None)
#Has data
assert train.shape[0] > 0, "Train data is empty"
print(train.shape)
def create_generator(args, data, DeepForest_config):
""" Create generators for training and validation.
"""
#Split training and test data - hardcoded paths set below.
_ , test = preprocess.split_training(data, DeepForest_config, experiment=None)
#Training Generator
generator = H5Generator(
test,
batch_size=args.batch_size,
DeepForest_config=DeepForest_config,
group_method="none",
name = "training"
)
return(generator)
def test_load_retraining_data_ablation(DeepForest_config):
DeepForest_config["training_proportion"] = 0.5
data = generators.load_retraining_data(DeepForest_config)
train, test = split_training(data, DeepForest_config, experiment=None)
print("Train shape {}".format(train.shape))
def test_load_retraining_data(DeepForest_config):
data = generators.load_retraining_data(DeepForest_config)
train, test = split_training(data, DeepForest_config, experiment=None)
print("Train shape {}".format(train.shape))