def config(): print("Configuring tfrecord tests") config = {} config["patch_size"] = 200 config["patch_overlap"] = 0.05 config["annotations_xml"] = get_data("OSBS_029.xml") config["rgb_dir"] = "data" config["annotations_file"] = "tests/data/OSBS_029.csv" config["path_to_raster"] =get_data("OSBS_029.tif") config["image-min-side"] = 800 config["backbone"] = "resnet50" #Create a clean config test data annotations = utilities.xml_to_annotations(xml_path=config["annotations_xml"]) annotations.to_csv("tests/data/tfrecords_OSBS_029.csv",index=False) annotations_file = preprocess.split_raster(path_to_raster=config["path_to_raster"], annotations_file="tests/data/tfrecords_OSBS_029.csv", base_dir= "tests/data/", patch_size=config["patch_size"], patch_overlap=config["patch_overlap"]) annotations_file.to_csv("tests/data/testfile_tfrecords.csv", index=False,header=False) class_file = utilities.create_classes("tests/data/testfile_tfrecords.csv") return config
def multi_annotations(): annotations = utilities.xml_to_annotations(get_data("SOAP_061.xml")) annotations.image_path = annotations.image_path.str.replace(".tif", ".png") annotations_file = get_data("testfile_multi.csv") annotations.to_csv(annotations_file, index=False, header=False) return annotations_file
def test_xml_to_annotations(): annotations = utilities.xml_to_annotations(xml_path = get_data("OSBS_029.xml")) print(annotations.shape) assert annotations.shape == (61 ,6) #bounding box extents should be int assert annotations["xmin"].dtype == "int"
def annotations(): annotations = utilities.xml_to_annotations(get_data("OSBS_029.xml")) # Point at the png version for tfrecords annotations.image_path = annotations.image_path.str.replace(".tif", ".png") annotations_file = get_data("testfile_deepforest.csv") annotations.to_csv(annotations_file, index=False, header=False) return annotations_file
def config(): config = utilities.read_config(get_data("deepforest_config.yml")) config["patch_size"] = 200 config["patch_overlap"] = 0.25 config["annotations_xml"] = get_data("OSBS_029.xml") config["rgb_dir"] = "data" config["annotations_file"] = "tests/data/OSBS_029.csv" config["path_to_raster"] = get_data("OSBS_029.tif") # Create a clean config test data annotations = utilities.xml_to_annotations(xml_path=config["annotations_xml"]) annotations.to_csv("tests/data/OSBS_029.csv", index=False) return config
def config(): config = {} config["patch_size"] = 200 config["patch_overlap"] = 0.25 config["annotations_xml"] = "tests/data/OSBS_029.xml" config["rgb_dir"] = "tests/data" config["annotations_file"] = "tests/data/OSBS_029.csv" config["path_to_raster"] = "tests/data/OSBS_029.tif" #Create a clean config test data annotations = utilities.xml_to_annotations( xml_path=config["annotations_xml"], rgb_dir=config["rgb_dir"]) annotations.to_csv("tests/data/OSBS_029.csv", index=False) return config
def test_split_raster(config, tmpdir): """Split raster into crops with overlaps to maintain all annotations""" raster = get_data("2019_YELL_2_528000_4978000_image_crop2.png") annotations = utilities.xml_to_annotations( get_data("2019_YELL_2_528000_4978000_image_crop2.xml")) annotations.to_csv("{}/example.csv".format(tmpdir), index=False) #annotations.label = 0 #visualize.plot_prediction_dataframe(df=annotations, root_dir=os.path.dirname(get_data(".")), show=True) annotations_file = preprocess.split_raster( path_to_raster=raster, annotations_file="{}/example.csv".format(tmpdir), base_dir=tmpdir, patch_size=500, patch_overlap=0) # Returns a 6 column pandas array assert annotations_file.shape[1] == 6
def generate_benchmark(BENCHMARK_PATH): tifs = glob.glob(BENCHMARK_PATH + "evaluation/RGB/*.tif") xmls = [os.path.splitext(os.path.basename(x))[0] for x in tifs] xmls = [ os.path.join(BENCHMARK_PATH, "annotations", x) + ".xml" for x in xmls ] #Load and format xmls, not every RGB image has an annotation annotation_list = [] for xml_path in xmls: try: annotation = utilities.xml_to_annotations(xml_path) except: continue annotation_list.append(annotation) benchmark_annotations = pd.concat(annotation_list, ignore_index=True) #save evaluation annotations fname = os.path.join(BENCHMARK_PATH + "evaluation/RGB/benchmark_annotations.csv") benchmark_annotations.to_csv(fname, index=False, header=None)
def run(rgb_dir, savedir): """Generate a DeepForest annotation file for a directory of xml and rgb tifs""" tifs = glob.glob(rgb_dir + "*.tif") xmls = [os.path.splitext(x)[0] for x in tifs] xmls = ["{}.xml".format(x) for x in xmls] if len(xmls) == 0: raise IOError("There are no matching .xml files in {}".format( os.path.abspath(rgb_dir))) #Load and format xmls, not every RGB image has an annotation annotation_list = [] for xml_path in xmls: try: annotation = utilities.xml_to_annotations(xml_path) annotation_list.append(annotation) except: pass benchmark_annotations = pd.concat(annotation_list, ignore_index=True) #save evaluation annotations fname = os.path.join(savedir + "benchmark_annotations.csv") benchmark_annotations.to_csv(fname, index=False, header=None) return fname
def test_float_warning(config): """Users should get a rounding warning when adding annotations with floats""" float_annotations = "tests/data/float_annotations.txt" annotations = utilities.xml_to_annotations(float_annotations) assert annotations.xmin.dtype is np.dtype('int64')
def annotations(): annotations = utilities.xml_to_annotations(get_data("OSBS_029.xml")) annotations_file = "tests/data/OSBS_029.csv" annotations.to_csv(annotations_file, index=False, header=False) return annotations_file
def annotations(): annotations = utilities.xml_to_annotations("tests/data/OSBS_029.xml", rgb_dir="tests/data") annotations.to_csv("tests/data/OSBS_029.csv", index=False, header=None)
def generate_training(DEBUG, BASE_PATH, dask_client=None, allow_empty=False): #Hand annotation dirname dirname = "hand_annotations/" #Remove previous files if needed previous_files = [ dirname + "crops/hand_annotations.csv", dirname + "hand_annotations.csv", dirname + "crops/classes.csv" ] for f in previous_files: full_path = os.path.join(BASE_PATH, f) if os.path.exists(full_path): os.remove(full_path) ## Hand annotations ## #convert hand annotations from xml into retinanet format xmls = glob.glob(BENCHMARK_PATH + "annotations/" + "*.xml") annotation_list = [] for xml in xmls: #check if it is in the directory image_name = "{}.tif".format( os.path.splitext(os.path.basename(xml))[0]) if os.path.exists(os.path.join(BASE_PATH + dirname + image_name)): print(xml) annotation = utilities.xml_to_annotations(xml) annotation_list.append(annotation) #Collect hand annotations annotations = pd.concat(annotation_list, ignore_index=True) #collect shapefile annotations shps = glob.glob(BASE_PATH + dirname + "*.shp") shps_tifs = glob.glob(BASE_PATH + dirname + "*.tif") shp_results = [] for shp in shps: rgb = "{}.tif".format(os.path.splitext(shp)[0]) shp_df = shapefile_to_annotations(shp, rgb) shp_df = pd.DataFrame(shp_df) shp_results.append(shp_df) shp_results = pd.concat(shp_results, ignore_index=True) annotations = pd.concat([annotations, shp_results]) #force dtype annotations.xmin = annotations.xmin.astype(int) annotations.ymin = annotations.ymin.astype(int) annotations.xmax = annotations.xmax.astype(int) annotations.ymax = annotations.ymax.astype(int) annotations.to_csv(BASE_PATH + dirname + "hand_annotations.csv", index=False) #Collect hand annotation tiles xmls = glob.glob(BASE_PATH + dirname + "*.xml") xmls = [os.path.splitext(os.path.basename(x))[0] for x in xmls] raster_list = [BASE_PATH + dirname + x + ".tif" for x in xmls] raster_list = raster_list + shps_tifs if DEBUG: raster_list = [raster_list[0]] cropped_annotations = [] if dask_client: futures = dask_client.map(preprocess.split_raster, raster_list, annotations_file=BASE_PATH + dirname + "hand_annotations.csv", base_dir=BASE_PATH + dirname + "crops/", patch_size=400, patch_overlap=0.05, allow_empty=allow_empty) wait(futures) for future in futures: try: local_annotations = future.result() cropped_annotations.append(local_annotations) except Exception as e: print("future {} failed with {}".format(future, e)) print("hand annotation generation complete") else: for raster in raster_list: annotations_df = preprocess.split_raster( path_to_raster=raster, annotations_file=BASE_PATH + dirname + "hand_annotations.csv", base_dir=BASE_PATH + dirname + "crops/", patch_size=400, patch_overlap=0.05) cropped_annotations.append(annotations_df) ##Gather annotation files into a single file annotations = pd.concat(cropped_annotations, ignore_index=True) #Ensure column order annotations.to_csv(BASE_PATH + dirname + "crops/hand_annotations.csv", index=False, header=None) print(annotations.head())
from deepforest import deepforest from deepforest import preprocess from deepforest import utilities from deepforest import tfrecords from deepforest import get_data annotations = utilities.xml_to_annotations(get_data("SOAP_061.xml")) annotations.image_path = annotations.image_path.str.replace(".tif", ".png") annotations_file = get_data("testfile_multi.csv") annotations.to_csv(annotations_file, index=False, header=False) test_model = deepforest.deepforest() #test_model.use_release() test_model.config["epochs"] = 1000 test_model.config["save-snapshot"] = False test_model.config["steps"] = 1 #test_model.config["multi_gpu"] = 1 test_model.config["save_path"] = "/home/b.weinstein/logs/deepforest" test_model.config["freeze_resnet"] = False test_model.train(annotations=annotations_file, input_type="fit_generator") # Test labels labels = list(test_model.labels.values()) labels.sort() target_labels = ["Dead", "Alive"] target_labels.sort() assert labels == target_labels image_path = get_data("SOAP_061.png") image = test_model.predict_image(image_path=image_path, return_plot=True)