def test_generate_tfrecords(train, tmpdir):
created_records = boxes.generate_tfrecords(shapefile=test_predictions,
savedir=tmpdir,
train=train,
sensor_path=test_sensor_tile,
height=20,
width=20,
classes=6)
assert all([os.path.exists(x) for x in created_records])
if train:
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="train")
else:
dataset = boxes.tf_dataset(created_records,
batch_size=2,
mode="predict")
if train:
for image_batch, label_batch in dataset.take(3):
assert image_batch.shape == (2, 20, 20, 3)
assert label_batch.shape == (2, 6)
else:
for image_batch, box_index_batch, in dataset.take(3):
assert image_batch.shape == (2, 20, 20, 3)
assert box_index_batch.shape == (2, )
def generate(self, shapefile, sensor_path, train=True, chunk_size=1000):
"""Predict species class for each DeepForest bounding box
Args:
shapefile: a DeepForest shapefile (see NeonCrownMaps) with a bounding box and utm projection
train: generate a training record that yields, image, label, or a prediction record with metadata? Default True
sensor_path: supply a known path to a sensor geoTIFF tile.
chunk_size: number of crops per tfrecord
"""
#set savedir
if train:
savedir = self.config["train"]["tfrecords"]
else:
savedir = self.config["predict"]["tfrecords"]
created_records = boxes.generate_tfrecords(shapefile=shapefile,
sensor_path=sensor_path,
height=self.height,
width=self.width,
savedir=savedir,
train=train,
classes=self.classes,
chunk_size=chunk_size,
extend_box=self.extend_box,
shuffle=True)
return created_records
def generate(shapefile,
savedir,
train=True,
sensor_path=None,
height=40,
width=40,
classes=20):
"""Predict species class for each DeepForest bounding box
Args:
shapefile: a DeepForest shapefile (see NeonCrownMaps) with a bounding box and utm projection
savedir: directory to save created tfrecords
train: generate a training record that yields, image, label, or a prediction record with metadata? Default True
sensor_path: supply a known path to a sensor tile. If not, use a lookup function hardcoded to a dir
"""
if sensor_path is None:
sensor_path = find_hyperspectral_path(shapefile)
created_records = boxes.generate_tfrecords(shapefile,
sensor_path,
height=height,
width=width,
savedir=savedir,
train=train,
classes=classes)
return created_records
def test_generate_tfrecords(train, tmpdir):
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
site = 1,
elevation=100,
savedir=tmpdir,
train=train,
HSI_sensor_path=test_sensor_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=20,
classes=6)
assert all([os.path.exists(x) for x in created_records])
if train:
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="ensemble")
else:
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="predict")
if train:
for (HSI, RGB), label_batch in dataset.take(3):
assert HSI.shape == (2,20,20,3)
assert RGB.shape == (2,100,100,3)
assert label_batch.shape == (2,6)
else:
for (HSI, RGB ), box_index_batch in dataset.take(3):
assert HSI.shape == (2,20,20,3)
assert RGB.shape == (2,100,100,3)
assert box_index_batch.shape == (2,)
def generate(self, shapefile, HSI_sensor_path, RGB_sensor_path, elevation, heights, site, species_label_dict=None, train=True, chunk_size=1000):
"""Predict species class for each DeepForest bounding box
Args:
shapefile: a DeepForest shapefile (see NeonCrownMaps) with a bounding box and utm projection
train: generate a training record that yields, image, label, or a prediction record with metadata? Default True
site: site metadata label
height: list of heights in the shapefile
sensor_path: supply a known path to a sensor geoTIFF tile.
chunk_size: number of crops per tfrecord
"""
#set savedir
if train:
savedir = self.config["train"]["tfrecords"]
else:
savedir = self.config["predict"]["tfrecords"]
self.classes = pd.read_csv(self.classes_file).shape[0]
created_records = boxes.generate_tfrecords(shapefile=shapefile,
HSI_sensor_path=HSI_sensor_path,
RGB_sensor_path=RGB_sensor_path,
site=site,
elevation=elevation,
heights=heights,
species_label_dict=species_label_dict,
HSI_size=self.HSI_size,
RGB_size=self.RGB_size,
savedir=savedir,
train=train,
number_of_sites=self.sites,
classes=self.classes,
chunk_size=chunk_size,
extend_box=self.extend_box,
shuffle=True)
return created_records
def test_generate_records(tmpdir, ensemble_model):
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
domain=1,
site=1,
elevation=100.0,
savedir=tmpdir,
HSI_sensor_path=test_hsi_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=10,
classes=6,
number_of_sites=10,
number_of_domains=10,
ensemble_model=None)
shp = gpd.read_file(test_predictions)
dataset = boxes.tf_dataset(created_records, batch_size=1)
counter = 0
for batch in dataset:
counter += 1
assert counter == shp.shape[0]
def generate(self,
HSI_sensor_path,
RGB_sensor_path,
elevation,
domain,
site,
savedir,
species_label_dict=None,
train=True,
chunk_size=1000,
shapefile=None,
csv_file=None,
label_column="label",
ensemble_model=None,
raw_boxes=None,
HSI_crop_savedir=None):
"""Predict species class for each DeepForest bounding box
Args:
shapefile: a shapefile (see NeonCrownMaps) with a bounding box in utm projection and label, columns -> xmin, ymin,xmax,ymax, label
train: generate a training record that yields, image, label, or a prediction record with metadata? Default True
site: site metadata label in numeric
sensor_path: supply a known path to a sensor geoTIFF tile.
chunk_size: number of crops per tfrecord
label_column: name of column to take taxonID labels
raw_boxes: .shp of neighbor boxes to select
"""
self.classes = pd.read_csv(self.classes_file).shape[0]
created_records = boxes.generate_tfrecords(
shapefile=shapefile,
csv_file=csv_file,
HSI_sensor_path=HSI_sensor_path,
RGB_sensor_path=RGB_sensor_path,
domain=domain,
site=site,
elevation=elevation,
species_label_dict=species_label_dict,
HSI_size=self.HSI_size,
RGB_size=self.RGB_size,
savedir=savedir,
train=train,
number_of_sites=self.sites,
number_of_domains=self.domains,
classes=self.classes,
chunk_size=chunk_size,
extend_HSI_box=self.config["train"]["HSI"]["extend_box"],
extend_RGB_box=self.config["train"]["RGB"]["extend_box"],
label_column=label_column,
shuffle=self.config["train"]["shuffle"],
ensemble_model=ensemble_model,
raw_boxes=raw_boxes,
k_neighbors=self.config["neighbors"]["k_neighbors"],
HSI_crop_savedir=self.config["HSI_crop_savedir"])
return created_records
def created_records(tmpdir):
shp = gpd.read_file(test_predictions)
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
domain=1,
site=1,
heights=np.random.random(shp.shape[0]) * 10,
elevation=100.0,
savedir=tmpdir,
HSI_sensor_path=test_hsi_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=40,
classes=6,
number_of_sites=10,
number_of_domains=16)
return created_records
def created_records(tmpdir, ensemble_model):
shp = gpd.read_file(test_predictions)
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
domain=1,
site=1,
elevation=100.0,
savedir=tmpdir,
HSI_sensor_path=test_hsi_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=10,
classes=6,
number_of_sites=10,
number_of_domains=10,
ensemble_model=None)
return created_records
def test_metadata(tmpdir):
shp = gpd.read_file(test_predictions)
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
site = 1,
heights=np.random.random(shp.shape[0])*10,
elevation=100,
savedir=tmpdir,
HSI_sensor_path=test_sensor_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=20,
classes=6,
number_of_sites=10)
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="metadata")
for data, label_batch in dataset.take(3):
elevation, height, site = data
assert elevation.numpy().shape == (2,)
assert site.numpy().shape == (2,10)
def test_generate_tfrecords(train, tmpdir):
shp = gpd.read_file(test_predictions)
created_records = boxes.generate_tfrecords(
shapefile=test_predictions,
site = 1,
elevation=100,
heights=np.random.random(shp.shape[0])*10,
savedir=tmpdir,
train=train,
HSI_sensor_path=test_sensor_tile,
RGB_sensor_path=test_sensor_tile,
species_label_dict=None,
RGB_size=100,
HSI_size=20,
classes=6,
number_of_sites=10)
assert all([os.path.exists(x) for x in created_records])
if train:
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="ensemble")
else:
dataset = boxes.tf_dataset(created_records, batch_size=2, mode="predict")
if train:
#Yield a batch of data and confirm its shape
for (HSI, RGB, elevation, height, site), label_batch in dataset.take(3):
assert HSI.shape == (2,20,20,3)
assert RGB.shape == (2,100,100,3)
assert elevation.shape == (2)
assert site.shape == (2,10)
assert height.shape == (2)
assert label_batch.shape == (2,6)
else:
for (HSI, RGB ), box_index_batch in dataset.take(3):
assert HSI.shape == (2,20,20,3)
assert RGB.shape == (2,100,100,3)
assert box_index_batch.shape == (2,)
