def train(pretrain_model_path, proportion_data, DeepForest_config):
###Log experiments
experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",
project_name='deeplidar',
log_code=False)
#make snapshot dir
dirname = datetime.now().strftime("%Y%m%d_%H%M%S")
experiment.log_parameter("Start Time", dirname)
save_snapshot_path = DeepForest_config["save_snapshot_path"] + dirname
os.mkdir(save_snapshot_path)
##Replace config file and experiment
DeepForest_config["batch_size"] = 40
DeepForest_config["epochs"] = 40
experiment.log_parameter("mode", "ablation")
DeepForest_config["evaluation_images"] = 0
#set training images, as a function of the number of training windows
DeepForest_config["training_proportion"] = proportion_data
experiment.log_parameters(DeepForest_config)
#Create model
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=1,
weights=pretrain_model_path,
multi_gpu=2,
freeze_backbone=False,
nms_threshold=DeepForest_config["nms_threshold"],
input_channels=DeepForest_config["input_channels"])
if not proportion_data == 0:
#Run training, and pass comet experiment class
#start training
data = load_retraining_data(DeepForest_config)
train_generator, validation_generator = create_h5_generators(
data, DeepForest_config=DeepForest_config)
#ensure directory created first; otherwise h5py will error after epoch.
history = training_model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.size() /
DeepForest_config["batch_size"],
epochs=DeepForest_config["epochs"],
verbose=2,
shuffle=False,
workers=DeepForest_config["workers"],
use_multiprocessing=DeepForest_config["use_multiprocessing"],
max_queue_size=DeepForest_config["max_queue_size"])
num_trees = train_generator.total_trees
else:
num_trees = 0
#Log trees
experiment.log_parameter("Number of Training Trees", num_trees)
return prediction_model, num_trees
def test_split_training(DeepForest_config):
data = generators.load_retraining_data(DeepForest_config)
train, test = preprocess.split_training(
data, DeepForest_config=DeepForest_config, experiment=None)
#Has data
assert train.shape[0] > 0, "Train data is empty"
print(train.shape)
def test_multigpu_training():
experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",
project_name='deeplidar',
log_code=True)
DeepForest_config = config.load_config(dir="..")
DeepForest_config["save_image_path"] = "../snapshots/"
data = generators.load_retraining_data(DeepForest_config)
train_generator, validation_generator = generators.create_h5_generators(
data, DeepForest_config=DeepForest_config)
#imagenet pretraining weights
backbone = models.backbone(DeepForest_config["backbone"])
weights = backbone.download_imagenet()
model, training_model, prediction_model = create_models(
backbone_retinanet=backbone.retinanet,
num_classes=train_generator.num_classes(),
weights=weights,
multi_gpu=DeepForest_config["num_GPUs"],
freeze_backbone=False,
nms_threshold=DeepForest_config["nms_threshold"],
input_channels=DeepForest_config["input_channels"])
#start training
history = training_model.fit_generator(
generator=train_generator,
steps_per_epoch=train_generator.size() /
DeepForest_config["batch_size"],
epochs=DeepForest_config["epochs"],
verbose=2,
shuffle=False,
workers=DeepForest_config["workers"],
use_multiprocessing=DeepForest_config["use_multiprocessing"],
max_queue_size=DeepForest_config["max_queue_size"],
experiment=experiment)
precision = 8
fp = open('h5_memory.log', 'w+')
#Path hack
dir_path = os.path.dirname(os.path.realpath(__file__))
parent_path = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
sys.path.append(parent_path)
print(parent_path)
from DeepForest import h5_generator, preprocess, config
from DeepForest.utils import generators, image_utils
DeepForest_config = config.load_config(dir="..")
#Load xml annotations
tiles = generators.load_retraining_data(DeepForest_config)
train, test = generators.split_tiles(tiles, DeepForest_config)
@profile(precision=precision, stream=fp)
def test_h5_generator(train, DeepForest_config):
#Training Generator
generator = h5_generator.H5generator(train)
for i in range(len(generator)):
inputs, targets = generator.__getitem__(i)
assert len(targets) == 2, "targets has incorrect length"
assert inputs.shape == (DeepForest_config["batch_size"],
DeepForest_config["patch_size"],
def test_load_retraining_data_ablation(DeepForest_config):
DeepForest_config["training_proportion"] = 0.5
data = generators.load_retraining_data(DeepForest_config)
train, test = split_training(data, DeepForest_config, experiment=None)
print("Train shape {}".format(train.shape))
def test_load_retraining_data(DeepForest_config):
data = generators.load_retraining_data(DeepForest_config)
train, test = split_training(data, DeepForest_config, experiment=None)
print("Train shape {}".format(train.shape))
#set experiment and log configs
experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",project_name='deeplidar',log_code=True)
DeepForest_config = load_config()
#Log parameters
experiment.log_parameter("Start Time", mode.dir)
experiment.log_parameter("Training Mode", mode.mode)
experiment.log_parameters(DeepForest_config)
DeepForest_config["mode"] = mode.mode
if mode.mode == "train":
data = load_training_data(DeepForest_config)
if mode.mode == "retrain":
data = load_retraining_data(DeepForest_config)
for x in DeepForest_config["evaluation_site"]:
DeepForest_config[x]["h5"] = os.path.join(DeepForest_config[x]["h5"],"hand_annotations")
#pass an args object instead of using command line
args = [
"--batch-size", str(DeepForest_config['batch_size']),
'--score-threshold', str(DeepForest_config['score_threshold']),
'--suppression-threshold', '0.1',
'--save-path', 'snapshots/images/',
'--model', mode.saved_model,
'--convert-model'
]
#Run training, and pass comet experiment
main(data, DeepForest_config, experiment, args)
def test_retrain():
data = load_retraining_data(DeepForest_config)
print("Data shape is {}".format(data.shape))
assert data.shape[0] > 0, "Data is empty"
