import pandas as pd
model = AttentionModel(config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml")
model.create()
#Log config
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.add_tag("HSI")
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
model.read_data()
class_weight = model.calc_class_weight()
##Train subnetwork
experiment.log_parameter("Train subnetworks", True)
with experiment.context_manager("HSI_spatial_subnetwork"):
print("Train HSI spatial subnetwork")
model.read_data(mode="HSI_submodel")
model.train(submodel="spatial", sensor="hyperspectral",class_weight=[class_weight, class_weight, class_weight], experiment=experiment)
with experiment.context_manager("HSI_spectral_subnetwork"):
print("Train HSI spectral subnetwork")
model.read_data(mode="HSI_submodel")
model.train(submodel="spectral", sensor="hyperspectral", class_weight=[class_weight, class_weight, class_weight], experiment=experiment)
#Train full model
#Create a class and run
model = AttentionModel(config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml", log_dir=save_dir)
model.create()
if model.config["train"]["pretraining_dir"]:
model.HSI_model.load_weights("{}/HSI_model.h5".format(model.config["train"]["pretraining_dir"]))
#Log config
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.log_parameters(model.config["train"]["ensemble"])
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
model.read_data(mode="HSI")
#Log the size of the training data
counter=0
for data, label in model.train_split:
counter += data.shape[0]
experiment.log_parameter("Training Samples", counter)
#Load from file and compile or train new models
if model.config["train"]["checkpoint_dir"] is not None:
dirname = model.config["train"]["checkpoint_dir"]
if model.config["train"]["gpus"] > 1:
with model.strategy.scope():
print("Running in parallel on {} GPUs".format(model.strategy.num_replicas_in_sync))
#model.RGB_model = load_model("{}/RGB_model.h5".format(dirname), custom_objects={"WeightedSum": WeightedSum}, compile=False)
model.HSI_model = load_model("{}/HSI_model.h5".format(dirname), custom_objects={"WeightedSum": WeightedSum}, compile=False)
import pandas as pd
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml")
model.create()
#Log config
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.add_tag("metadata")
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
model.read_data(mode="metadata")
#Cree
inputs, outputs = metadata.metadata_model(
classes=model.config["train"]["classes"])
meta_model = tf.keras.Model(inputs=inputs,
outputs=outputs,
name="DeepTreeAttention")
meta_model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=["acc"])
labeldf = pd.read_csv(model.classes_file)
callback_list = callbacks.ConfusionMatrixCallback(experiment,
model.val_split,
save_dir = "{}/{}".format("/orange/idtrees-collab/DeepTreeAttention/snapshots/",timestamp)
os.mkdir(save_dir)
experiment.log_parameter("timestamp",timestamp)
#Create a class and run
model = AttentionModel(config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml")
model.create()
#Log config
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["predict"])
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
model.read_data(validation_split=True)
class_weight = model.calc_class_weight()
## Train subnetwork
experiment.log_parameter("Train subnetworks", True)
with experiment.context_manager("spatial_subnetwork"):
print("Train spatial subnetwork")
model.read_data(mode="submodel",validation_split=True)
model.train(submodel="spatial", class_weight=[class_weight, class_weight, class_weight])
with experiment.context_manager("spectral_subnetwork"):
print("Train spectral subnetwork")
model.read_data(mode="submodel",validation_split=True)
model.train(submodel="spectral", class_weight=[class_weight, class_weight, class_weight])
#Train full model
experiment.log_parameter("Class Weighted", True)
from time import sleep
from random import randint
from datetime import datetime
import os
from comet_ml import Experiment
from DeepTreeAttention.trees import AttentionModel
sleep(randint(0,20))
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
save_dir = "{}/{}".format("/orange/idtrees-collab/DeepTreeAttention/snapshots/",timestamp)
os.mkdir(save_dir)
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.add_tag("Cleaning")
#Create output folder
experiment.log_parameter("timestamp",timestamp)
experiment.log_parameter("log_dir",save_dir)
#Create a class and run
model = AttentionModel(config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml", log_dir=save_dir)
model.read_data("HSI_autoencoder")
train_df, test_df = model.find_outliers()
train_df.to_file("{}/train_outliers.shp".format(save_dir))
test_df.to_file("{}/test_outliers.shp".format(save_dir))
save_dir = "{}/{}".format(
"/orange/idtrees-collab/DeepTreeAttention/snapshots/", timestamp)
os.mkdir(save_dir)
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.add_tag("Train")
#Create output folder
experiment.log_parameter("timestamp", timestamp)
experiment.log_parameter("log_dir", save_dir)
#Create a class and run
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml",
log_dir=save_dir)
model.read_data("HSI")
model.create()
baseline = vanilla.create(
height=model.config["train"]["HSI"]["crop_size"],
width=model.config["train"]["HSI"]["crop_size"],
channels=model.config["train"]["HSI"]["sensor_channels"],
classes=model.classes)
baseline.compile(loss="categorical_crossentropy",
optimizer=tf.keras.optimizers.Adam(
lr=float(model.config["train"]["learning_rate"])),
metrics=[tf.keras.metrics.CategoricalAccuracy(name='acc')])
labeldf = pd.read_csv(model.classes_file)
label_names = list(labeldf.taxonID.values)
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.add_tag("neighbors")
#Create output folder
experiment.log_parameter("timestamp", timestamp)
experiment.log_parameter("log_dir", save_dir)
#Create a class and run
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml",
log_dir=save_dir)
model.create()
ensemble_model = tfk.models.load_model(
"{}/Ensemble.h5".format(model.config["neighbors"]["model_dir"]),
custom_objects={"WeightedSum": WeightedSum})
model.read_data("neighbors")
experiment.log_parameters(model.config["neighbors"])
neighbor = neighbors_model.create(
ensemble_model=ensemble_model,
k_neighbors=model.config["neighbors"]["k_neighbors"],
classes=model.classes,
freeze=model.config["neighbors"]["freeze"])
labeldf = pd.read_csv(model.classes_file)
label_names = list(labeldf.taxonID.values)
callback_list = callbacks.create(experiment=experiment,
train_data=model.train_split,
validation_data=model.val_split,
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml")
model.create()
#Log config
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.add_tag("RGB")
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
model.read_data()
class_weight = model.calc_class_weight()
##Train subnetwork
experiment.log_parameter("Train subnetworks", True)
with experiment.context_manager("RGB_spatial_subnetwork"):
print("Train RGB spatial subnetwork")
model.read_data(mode="RGB_submodel")
model.train(submodel="spatial",
sensor="RGB",
class_weight=[class_weight, class_weight, class_weight],
experiment=experiment)
with experiment.context_manager("RGB_spectral_subnetwork"):
print("Train RGB spectral subnetwork")
model.read_data(mode="RGB_submodel")
#Linear metadata model for testing purposes
from comet_ml import Experiment
import tensorflow as tf
from DeepTreeAttention.trees import AttentionModel
from DeepTreeAttention.models import metadata
from DeepTreeAttention.callbacks import callbacks
import pandas as pd
model = AttentionModel(
config="/home/b.weinstein/DeepTreeAttention/conf/tree_config.yml")
model.create()
#Log config
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.add_tag("metadata")
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
with experiment.context_manager("metadata"):
model.read_data(HSI=False, RGB=False, metadata=True)
class_weight = model.calc_class_weight()
model.train(submodel="metadata",
experiment=experiment,
class_weight=class_weight)
#Log config
experiment = Experiment(project_name="neontrees", workspace="bw4sz")
experiment.log_parameters(model.config["train"])
experiment.log_parameters(model.config["evaluation"])
experiment.log_parameters(model.config["predict"])
experiment.add_tag("RGB")
experiment.log_parameter("timestamp", timestamp)
##Train
#Train see config.yml for tfrecords path with weighted classes in cross entropy
##Train subnetwork
experiment.log_parameter("Train subnetworks", True)
with experiment.context_manager("RGB_spatial_subnetwork"):
print("Train RGB spatial subnetwork")
model.read_data(HSI=False, RGB=True, metadata=False, submodel=False)
model.train(submodel="spatial", sensor="RGB", experiment=experiment)
with experiment.context_manager("RGB_spectral_subnetwork"):
print("Train RGB spectral subnetwork")
model.train(submodel="spectral", sensor="RGB", experiment=experiment)
#Train full model
with experiment.context_manager("RGB_model"):
model.read_data(HSI=False, RGB=True, metadata=False)
model.train(sensor="RGB", experiment=experiment)
#Get Alpha score for the weighted spectral/spatial average. Higher alpha favors spatial network.
if model.config["train"]["RGB"]["weighted_sum"]:
estimate_a = model.RGB_model.get_layer("weighted_sum").get_weights()
experiment.log_metric(name="spatial-spectral weight",