def train(pretrain_model_path, proportion_data, DeepForest_config): ###Log experiments experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2", project_name='deeplidar', log_code=False) #make snapshot dir dirname = datetime.now().strftime("%Y%m%d_%H%M%S") experiment.log_parameter("Start Time", dirname) save_snapshot_path = DeepForest_config["save_snapshot_path"] + dirname os.mkdir(save_snapshot_path) ##Replace config file and experiment DeepForest_config["batch_size"] = 40 DeepForest_config["epochs"] = 40 experiment.log_parameter("mode", "ablation") DeepForest_config["evaluation_images"] = 0 #set training images, as a function of the number of training windows DeepForest_config["training_proportion"] = proportion_data experiment.log_parameters(DeepForest_config) #Create model model, training_model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=1, weights=pretrain_model_path, multi_gpu=2, freeze_backbone=False, nms_threshold=DeepForest_config["nms_threshold"], input_channels=DeepForest_config["input_channels"]) if not proportion_data == 0: #Run training, and pass comet experiment class #start training data = load_retraining_data(DeepForest_config) train_generator, validation_generator = create_h5_generators( data, DeepForest_config=DeepForest_config) #ensure directory created first; otherwise h5py will error after epoch. history = training_model.fit_generator( generator=train_generator, steps_per_epoch=train_generator.size() / DeepForest_config["batch_size"], epochs=DeepForest_config["epochs"], verbose=2, shuffle=False, workers=DeepForest_config["workers"], use_multiprocessing=DeepForest_config["use_multiprocessing"], max_queue_size=DeepForest_config["max_queue_size"]) num_trees = train_generator.total_trees else: num_trees = 0 #Log trees experiment.log_parameter("Number of Training Trees", num_trees) return prediction_model, num_trees
def test_split_training(DeepForest_config): data = generators.load_retraining_data(DeepForest_config) train, test = preprocess.split_training( data, DeepForest_config=DeepForest_config, experiment=None) #Has data assert train.shape[0] > 0, "Train data is empty" print(train.shape)
def test_multigpu_training(): experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2", project_name='deeplidar', log_code=True) DeepForest_config = config.load_config(dir="..") DeepForest_config["save_image_path"] = "../snapshots/" data = generators.load_retraining_data(DeepForest_config) train_generator, validation_generator = generators.create_h5_generators( data, DeepForest_config=DeepForest_config) #imagenet pretraining weights backbone = models.backbone(DeepForest_config["backbone"]) weights = backbone.download_imagenet() model, training_model, prediction_model = create_models( backbone_retinanet=backbone.retinanet, num_classes=train_generator.num_classes(), weights=weights, multi_gpu=DeepForest_config["num_GPUs"], freeze_backbone=False, nms_threshold=DeepForest_config["nms_threshold"], input_channels=DeepForest_config["input_channels"]) #start training history = training_model.fit_generator( generator=train_generator, steps_per_epoch=train_generator.size() / DeepForest_config["batch_size"], epochs=DeepForest_config["epochs"], verbose=2, shuffle=False, workers=DeepForest_config["workers"], use_multiprocessing=DeepForest_config["use_multiprocessing"], max_queue_size=DeepForest_config["max_queue_size"], experiment=experiment)
precision = 8 fp = open('h5_memory.log', 'w+') #Path hack dir_path = os.path.dirname(os.path.realpath(__file__)) parent_path = os.path.abspath(os.path.join(os.getcwd(), os.pardir)) sys.path.append(parent_path) print(parent_path) from DeepForest import h5_generator, preprocess, config from DeepForest.utils import generators, image_utils DeepForest_config = config.load_config(dir="..") #Load xml annotations tiles = generators.load_retraining_data(DeepForest_config) train, test = generators.split_tiles(tiles, DeepForest_config) @profile(precision=precision, stream=fp) def test_h5_generator(train, DeepForest_config): #Training Generator generator = h5_generator.H5generator(train) for i in range(len(generator)): inputs, targets = generator.__getitem__(i) assert len(targets) == 2, "targets has incorrect length" assert inputs.shape == (DeepForest_config["batch_size"], DeepForest_config["patch_size"],
def test_load_retraining_data_ablation(DeepForest_config): DeepForest_config["training_proportion"] = 0.5 data = generators.load_retraining_data(DeepForest_config) train, test = split_training(data, DeepForest_config, experiment=None) print("Train shape {}".format(train.shape))
def test_load_retraining_data(DeepForest_config): data = generators.load_retraining_data(DeepForest_config) train, test = split_training(data, DeepForest_config, experiment=None) print("Train shape {}".format(train.shape))
#set experiment and log configs experiment = Experiment(api_key="ypQZhYfs3nSyKzOfz13iuJpj2",project_name='deeplidar',log_code=True) DeepForest_config = load_config() #Log parameters experiment.log_parameter("Start Time", mode.dir) experiment.log_parameter("Training Mode", mode.mode) experiment.log_parameters(DeepForest_config) DeepForest_config["mode"] = mode.mode if mode.mode == "train": data = load_training_data(DeepForest_config) if mode.mode == "retrain": data = load_retraining_data(DeepForest_config) for x in DeepForest_config["evaluation_site"]: DeepForest_config[x]["h5"] = os.path.join(DeepForest_config[x]["h5"],"hand_annotations") #pass an args object instead of using command line args = [ "--batch-size", str(DeepForest_config['batch_size']), '--score-threshold', str(DeepForest_config['score_threshold']), '--suppression-threshold', '0.1', '--save-path', 'snapshots/images/', '--model', mode.saved_model, '--convert-model' ] #Run training, and pass comet experiment main(data, DeepForest_config, experiment, args)
def test_retrain(): data = load_retraining_data(DeepForest_config) print("Data shape is {}".format(data.shape)) assert data.shape[0] > 0, "Data is empty"