def config():
print("Configuring tfrecord tests")
config = {}
config["patch_size"] = 200
config["patch_overlap"] = 0.05
config["annotations_xml"] = get_data("OSBS_029.xml")
config["rgb_dir"] = "data"
config["annotations_file"] = "tests/data/OSBS_029.csv"
config["path_to_raster"] =get_data("OSBS_029.tif")
config["image-min-side"] = 800
config["backbone"] = "resnet50"
#Create a clean config test data
annotations = utilities.xml_to_annotations(xml_path=config["annotations_xml"])
annotations.to_csv("tests/data/tfrecords_OSBS_029.csv",index=False)
annotations_file = preprocess.split_raster(path_to_raster=config["path_to_raster"],
annotations_file="tests/data/tfrecords_OSBS_029.csv",
base_dir= "tests/data/",
patch_size=config["patch_size"],
patch_overlap=config["patch_overlap"])
annotations_file.to_csv("tests/data/testfile_tfrecords.csv", index=False,header=False)
class_file = utilities.create_classes("tests/data/testfile_tfrecords.csv")
return config
def multi_annotations():
annotations = utilities.xml_to_annotations(get_data("SOAP_061.xml"))
annotations.image_path = annotations.image_path.str.replace(".tif", ".png")
annotations_file = get_data("testfile_multi.csv")
annotations.to_csv(annotations_file, index=False, header=False)
return annotations_file
def test_xml_to_annotations():
annotations = utilities.xml_to_annotations(xml_path = get_data("OSBS_029.xml"))
print(annotations.shape)
assert annotations.shape == (61 ,6)
#bounding box extents should be int
assert annotations["xmin"].dtype == "int"
def annotations():
annotations = utilities.xml_to_annotations(get_data("OSBS_029.xml"))
# Point at the png version for tfrecords
annotations.image_path = annotations.image_path.str.replace(".tif", ".png")
annotations_file = get_data("testfile_deepforest.csv")
annotations.to_csv(annotations_file, index=False, header=False)
return annotations_file
def config():
config = utilities.read_config(get_data("deepforest_config.yml"))
config["patch_size"] = 200
config["patch_overlap"] = 0.25
config["annotations_xml"] = get_data("OSBS_029.xml")
config["rgb_dir"] = "data"
config["annotations_file"] = "tests/data/OSBS_029.csv"
config["path_to_raster"] = get_data("OSBS_029.tif")
# Create a clean config test data
annotations = utilities.xml_to_annotations(xml_path=config["annotations_xml"])
annotations.to_csv("tests/data/OSBS_029.csv", index=False)
return config
def config():
config = {}
config["patch_size"] = 200
config["patch_overlap"] = 0.25
config["annotations_xml"] = "tests/data/OSBS_029.xml"
config["rgb_dir"] = "tests/data"
config["annotations_file"] = "tests/data/OSBS_029.csv"
config["path_to_raster"] = "tests/data/OSBS_029.tif"
#Create a clean config test data
annotations = utilities.xml_to_annotations(
xml_path=config["annotations_xml"], rgb_dir=config["rgb_dir"])
annotations.to_csv("tests/data/OSBS_029.csv", index=False)
return config
def test_split_raster(config, tmpdir):
"""Split raster into crops with overlaps to maintain all annotations"""
raster = get_data("2019_YELL_2_528000_4978000_image_crop2.png")
annotations = utilities.xml_to_annotations(
get_data("2019_YELL_2_528000_4978000_image_crop2.xml"))
annotations.to_csv("{}/example.csv".format(tmpdir), index=False)
#annotations.label = 0
#visualize.plot_prediction_dataframe(df=annotations, root_dir=os.path.dirname(get_data(".")), show=True)
annotations_file = preprocess.split_raster(
path_to_raster=raster,
annotations_file="{}/example.csv".format(tmpdir),
base_dir=tmpdir,
patch_size=500,
patch_overlap=0)
# Returns a 6 column pandas array
assert annotations_file.shape[1] == 6
def generate_benchmark(BENCHMARK_PATH):
tifs = glob.glob(BENCHMARK_PATH + "evaluation/RGB/*.tif")
xmls = [os.path.splitext(os.path.basename(x))[0] for x in tifs]
xmls = [
os.path.join(BENCHMARK_PATH, "annotations", x) + ".xml" for x in xmls
]
#Load and format xmls, not every RGB image has an annotation
annotation_list = []
for xml_path in xmls:
try:
annotation = utilities.xml_to_annotations(xml_path)
except:
continue
annotation_list.append(annotation)
benchmark_annotations = pd.concat(annotation_list, ignore_index=True)
#save evaluation annotations
fname = os.path.join(BENCHMARK_PATH +
"evaluation/RGB/benchmark_annotations.csv")
benchmark_annotations.to_csv(fname, index=False, header=None)
def run(rgb_dir, savedir):
"""Generate a DeepForest annotation file for a directory of xml and rgb tifs"""
tifs = glob.glob(rgb_dir + "*.tif")
xmls = [os.path.splitext(x)[0] for x in tifs]
xmls = ["{}.xml".format(x) for x in xmls]
if len(xmls) == 0:
raise IOError("There are no matching .xml files in {}".format(
os.path.abspath(rgb_dir)))
#Load and format xmls, not every RGB image has an annotation
annotation_list = []
for xml_path in xmls:
try:
annotation = utilities.xml_to_annotations(xml_path)
annotation_list.append(annotation)
except:
pass
benchmark_annotations = pd.concat(annotation_list, ignore_index=True)
#save evaluation annotations
fname = os.path.join(savedir + "benchmark_annotations.csv")
benchmark_annotations.to_csv(fname, index=False, header=None)
return fname
def test_float_warning(config):
"""Users should get a rounding warning when adding annotations with floats"""
float_annotations = "tests/data/float_annotations.txt"
annotations = utilities.xml_to_annotations(float_annotations)
assert annotations.xmin.dtype is np.dtype('int64')
def annotations():
annotations = utilities.xml_to_annotations(get_data("OSBS_029.xml"))
annotations_file = "tests/data/OSBS_029.csv"
annotations.to_csv(annotations_file, index=False, header=False)
return annotations_file
def annotations():
annotations = utilities.xml_to_annotations("tests/data/OSBS_029.xml",
rgb_dir="tests/data")
annotations.to_csv("tests/data/OSBS_029.csv", index=False, header=None)
def generate_training(DEBUG, BASE_PATH, dask_client=None, allow_empty=False):
#Hand annotation dirname
dirname = "hand_annotations/"
#Remove previous files if needed
previous_files = [
dirname + "crops/hand_annotations.csv",
dirname + "hand_annotations.csv", dirname + "crops/classes.csv"
]
for f in previous_files:
full_path = os.path.join(BASE_PATH, f)
if os.path.exists(full_path):
os.remove(full_path)
## Hand annotations ##
#convert hand annotations from xml into retinanet format
xmls = glob.glob(BENCHMARK_PATH + "annotations/" + "*.xml")
annotation_list = []
for xml in xmls:
#check if it is in the directory
image_name = "{}.tif".format(
os.path.splitext(os.path.basename(xml))[0])
if os.path.exists(os.path.join(BASE_PATH + dirname + image_name)):
print(xml)
annotation = utilities.xml_to_annotations(xml)
annotation_list.append(annotation)
#Collect hand annotations
annotations = pd.concat(annotation_list, ignore_index=True)
#collect shapefile annotations
shps = glob.glob(BASE_PATH + dirname + "*.shp")
shps_tifs = glob.glob(BASE_PATH + dirname + "*.tif")
shp_results = []
for shp in shps:
rgb = "{}.tif".format(os.path.splitext(shp)[0])
shp_df = shapefile_to_annotations(shp, rgb)
shp_df = pd.DataFrame(shp_df)
shp_results.append(shp_df)
shp_results = pd.concat(shp_results, ignore_index=True)
annotations = pd.concat([annotations, shp_results])
#force dtype
annotations.xmin = annotations.xmin.astype(int)
annotations.ymin = annotations.ymin.astype(int)
annotations.xmax = annotations.xmax.astype(int)
annotations.ymax = annotations.ymax.astype(int)
annotations.to_csv(BASE_PATH + dirname + "hand_annotations.csv",
index=False)
#Collect hand annotation tiles
xmls = glob.glob(BASE_PATH + dirname + "*.xml")
xmls = [os.path.splitext(os.path.basename(x))[0] for x in xmls]
raster_list = [BASE_PATH + dirname + x + ".tif" for x in xmls]
raster_list = raster_list + shps_tifs
if DEBUG:
raster_list = [raster_list[0]]
cropped_annotations = []
if dask_client:
futures = dask_client.map(preprocess.split_raster,
raster_list,
annotations_file=BASE_PATH + dirname +
"hand_annotations.csv",
base_dir=BASE_PATH + dirname + "crops/",
patch_size=400,
patch_overlap=0.05,
allow_empty=allow_empty)
wait(futures)
for future in futures:
try:
local_annotations = future.result()
cropped_annotations.append(local_annotations)
except Exception as e:
print("future {} failed with {}".format(future, e))
print("hand annotation generation complete")
else:
for raster in raster_list:
annotations_df = preprocess.split_raster(
path_to_raster=raster,
annotations_file=BASE_PATH + dirname + "hand_annotations.csv",
base_dir=BASE_PATH + dirname + "crops/",
patch_size=400,
patch_overlap=0.05)
cropped_annotations.append(annotations_df)
##Gather annotation files into a single file
annotations = pd.concat(cropped_annotations, ignore_index=True)
#Ensure column order
annotations.to_csv(BASE_PATH + dirname + "crops/hand_annotations.csv",
index=False,
header=None)
print(annotations.head())
from deepforest import deepforest
from deepforest import preprocess
from deepforest import utilities
from deepforest import tfrecords
from deepforest import get_data
annotations = utilities.xml_to_annotations(get_data("SOAP_061.xml"))
annotations.image_path = annotations.image_path.str.replace(".tif", ".png")
annotations_file = get_data("testfile_multi.csv")
annotations.to_csv(annotations_file, index=False, header=False)
test_model = deepforest.deepforest()
#test_model.use_release()
test_model.config["epochs"] = 1000
test_model.config["save-snapshot"] = False
test_model.config["steps"] = 1
#test_model.config["multi_gpu"] = 1
test_model.config["save_path"] = "/home/b.weinstein/logs/deepforest"
test_model.config["freeze_resnet"] = False
test_model.train(annotations=annotations_file, input_type="fit_generator")
# Test labels
labels = list(test_model.labels.values())
labels.sort()
target_labels = ["Dead", "Alive"]
target_labels.sort()
assert labels == target_labels
image_path = get_data("SOAP_061.png")
image = test_model.predict_image(image_path=image_path, return_plot=True)
