def __get_metric(self, metric):
if metric == "categorical_accuracy":
return m.TopKCategoricalAccuracy(self.k)
elif metric == "sparse_categorical_accuracy":
return m.SparseTopKCategoricalAccuracy(self.k)
else:
raise Exception("specified metric not defined")
def initialize_model():
model = models.Sequential()
model.add(
layers.Convolution2D(16, (3, 3),
padding='same',
input_shape=(28, 28, 1),
activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Convolution2D(32, (3, 3), padding='same', activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(
layers.Convolution2D(64, (3, 3), padding='same', activation='relu'))
model.add(layers.MaxPooling2D(pool_size=(2, 2)))
model.add(layers.Flatten())
model.add(layers.Dense(128, activation='relu'))
model.add(layers.Dense(NUM_CLASSES, activation='softmax'))
# Train model
model.summary()
#adam = tf.compat.v1.train.AdamOptimizer()
f1 = tfa.metrics.F1Score(num_classes=NUM_CLASSES, average='macro')
kappa = tfa.metrics.CohenKappa(num_classes=NUM_CLASSES)
top = metrics.TopKCategoricalAccuracy(k=1)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy', kappa, f1, top])
return model
def test_evaluate(test_config):
#Create class
mod = Houston2018.AttentionModel(config="conf/houston_config.yml")
#Replace config for testing env
for key, value in test_config.items():
for nested_key, nested_value in value.items():
mod.config[key][nested_key] = nested_value
#Create
mod.create()
mod.read_data(validation_split=True)
metric_list = [
keras_metrics.TopKCategoricalAccuracy(k=2, name="top_k"),
keras_metrics.CategoricalAccuracy(name="acc")
]
mod.model.compile(loss="categorical_crossentropy",
optimizer=tf.keras.optimizers.Adam(
lr=float(mod.config['train']['learning_rate'])),
metrics=metric_list)
#Method 1, class eval method
print("Before evaluation")
y_pred, y_true = mod.evaluate(mod.val_split)
print("evaluated")
test_acc = keras_metrics.CategoricalAccuracy()
test_acc.update_state(y_true=y_true, y_pred=y_pred)
method1_eval_accuracy = test_acc.result().numpy()
assert y_pred.shape == y_true.shape
#Method 2, keras eval method
metric_list = mod.model.evaluate(mod.val_split)
metric_dict = {}
for index, value in enumerate(metric_list):
metric_dict[mod.model.metrics_names[index]] = value
assert method1_eval_accuracy == metric_dict["acc"]
#F1 requires integer, not softmax
f1s = metrics.f1_scores(y_true, y_pred)
def create(self, name="Hang2020",weights=None, submodel=None):
"""Load a model
Args:
weights: a saved model weights from previous run
name: a model name from DeepTreeAttention.models
"""
self.model = self.get_model(name)
if weights:
self.model.load_weights(weights)
#metrics
metric_list = [
metrics.TopKCategoricalAccuracy(k=2, name="top_k"),
metrics.CategoricalAccuracy(name="acc")]
#compile
self.model.compile(loss="categorical_crossentropy",
optimizer=tf.keras.optimizers.Adam(
lr=float(self.config['train']['learning_rate'])),
metrics=metric_list)
from tensorflow.keras.layers import GlobalAveragePooling1D, Flatten
from tensorflow.keras.layers import Dense, Activation, Dropout
from tensorflow.keras.activations import swish
from tensorflow_addons.losses import SigmoidFocalCrossEntropy as Focal
NAME = 'SSBML-Base-Model'
ACTIVATION = swish
OPTIMIZER = keras.optimizers.Adam()
LOSS = Focal()
METRICS = [
metrics.CategoricalAccuracy(name='accuracy'),
metrics.TopKCategoricalAccuracy(k=8, name='top 8 accuracy'),
]
def base_model(
name = NAME,
activation = ACTIVATION,
optimizer = OPTIMIZER,
loss = LOSS,
metrics = METRICS,
):
'''
Creates SSBML-Base-Model architecture
and compiles the model with the given
optimizer, loss, and metrics.
Parameters
model = create_model(X_lidar_train.shape[1:], X_coord_train.shape[1:],
num_classes)
model.summary(line_length=128)
# ## Training
K.clear_session()
optim = optimizers.Adam()
model_metrics = [
metrics.CategoricalAccuracy(),
metrics.TopKCategoricalAccuracy()
]
model_loss = losses.CategoricalCrossentropy()
model.compile(loss=model_loss, optimizer=optim, metrics=model_metrics)
# lrfinder = LearningRateFinder(model)
# lrfinder.find(train_generator, 1e-6, 1e0, 1)
# lrfinder.plot_loss()
# K.set_value(model.optimizer.lr, 1e-2)
EPOCHS = 50
if infinite:
data = data.repeat()
return data.prefetch(AUTOTUNE)
METRICS = [
# per_class_accuracy,
metrics.TruePositives(name='tp'),
metrics.FalsePositives(name='fp'),
metrics.TrueNegatives(name='tn'),
metrics.FalseNegatives(name='fn'),
metrics.CategoricalAccuracy(name='accuracy'),
metrics.Precision(name='precision'),
metrics.Recall(name='recall'),
metrics.TopKCategoricalAccuracy(name='top_3_categorical_accuracy', k=3),
metrics.TopKCategoricalAccuracy(name='top_5_categorical_accuracy', k=5)
]
###########################################################################
###########################################################################
encoder = base_dataset.LabelEncoder(data.data.family)
split_data = base_dataset.preprocess_data(data, encoder, data_config)
for subset, subset_data in split_data.items():
split_data[subset] = [list(i) for i in unzip(subset_data)]
steps_per_epoch=len(split_data['train'][0])//PARAMS['batch_size']
validation_steps=len(split_data['val'][0])//PARAMS['batch_size']
# split_datasets = {
def main(**kwargs):
import sys
for k, v in kwargs.items():
sys.argv += [k, v]
from pprint import pprint
import argparse
import datetime
import json
import os
parser = argparse.ArgumentParser()
parser.add_argument('--neptune_project_name', default='jacobarose/sandbox', type=str, help='Neptune.ai project name to log under')
parser.add_argument('--experiment_name', default='pnas_minimal_example', type=str, help='Neptune.ai experiment name to log under')
parser.add_argument('--config_path', default=r'/home/jacob/projects/pyleaves/pyleaves/configs/example_configs/pnas_resnet_config.json', type=str, help='JSON config file')
parser.add_argument('-gpu', '--gpu_id', default='1', type=str, help='integer number of gpu to train on', dest='gpu_id')
parser.add_argument('-tags', '--add-tags', default=[], type=str, nargs='*', help='Add arbitrary list of tags to apply to this run in neptune', dest='tags')
parser.add_argument('-f', default=None)
args = parser.parse_args()
with open(args.config_path, 'r') as config_file:
PARAMS = json.load(config_file)
# print(gpu)
# os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu_id)
pprint(PARAMS)
import tensorflow as tf
import neptune
# tf.debugging.set_log_device_placement(True)
print(tf.__version__)
import arrow
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import io
from stuf import stuf
from more_itertools import unzip
from functools import partial
# import tensorflow as tf
# tf.compat.v1.enable_eager_execution()
AUTOTUNE = tf.data.experimental.AUTOTUNE
from pyleaves.leavesdb.tf_utils.tf_utils import set_random_seed, reset_keras_session
import pyleaves
from pyleaves.utils.img_utils import random_pad_image
from pyleaves.utils.utils import ensure_dir_exists
from pyleaves.datasets import leaves_dataset, fossil_dataset, pnas_dataset, base_dataset
from pyleaves.models.vgg16 import VGG16, VGG16GrayScale
from pyleaves.models import resnet, vgg16
from tensorflow.compat.v1.keras.callbacks import Callback, ModelCheckpoint, TensorBoard, LearningRateScheduler, EarlyStopping
from tensorflow.keras import metrics
from tensorflow.keras.preprocessing.image import load_img, img_to_array
from tensorflow.keras import layers
from tensorflow.keras import backend as K
import tensorflow_datasets as tfds
import neptune_tensorboard as neptune_tb
seed = 346
# set_random_seed(seed)
# reset_keras_session()
def get_preprocessing_func(model_name):
if model_name.startswith('resnet'):
from tensorflow.keras.applications.resnet_v2 import preprocess_input
elif model_name == 'vgg16':
from tensorflow.keras.applications.vgg16 import preprocess_input
elif model_name=='shallow':
def preprocess_input(x):
return x/255.0 # ((x/255.0)-0.5)*2.0
return preprocess_input #lambda x,y: (preprocess_input(x),y)
def _load_img(image_path):#, img_size=(224,224)):
img = tf.io.read_file(image_path)
img = tf.image.decode_jpeg(img, channels=3)
img = tf.image.convert_image_dtype(img, tf.float32)
return img
# return tf.compat.v1.image.resize_image_with_pad(img, *img_size)
def _encode_label(label, num_classes=19):
label = tf.cast(label, tf.int32)
label = tf.one_hot(label, depth=num_classes)
return label
def _load_example(image_path, label, num_classes=19):
img = _load_img(image_path)
one_hot_label = _encode_label(label, num_classes=num_classes)
return img, one_hot_label
def _load_uint8_example(image_path, label, num_classes=19):
img = tf.image.convert_image_dtype(_load_img(image_path)*255.0, dtype=tf.uint8)
one_hot_label = _encode_label(label, num_classes=num_classes)
return img, one_hot_label
def rgb2gray_3channel(img, label):
'''
Convert rgb image to grayscale, but keep num_channels=3
'''
img = tf.image.rgb_to_grayscale(img)
img = tf.image.grayscale_to_rgb(img)
return img, label
def rgb2gray_1channel(img, label):
'''
Convert rgb image to grayscale, num_channels from 3 to 1
'''
img = tf.image.rgb_to_grayscale(img)
return img, label
def log_data(logs):
for k, v in logs.items():
neptune.log_metric(k, v)
neptune_logger = tf.keras.callbacks.LambdaCallback(on_epoch_end=lambda epoch, logs: log_data(logs))
def focal_loss(gamma=2.0, alpha=4.0):
gamma = float(gamma)
alpha = float(alpha)
def focal_loss_fixed(y_true, y_pred):
"""Focal loss for multi-classification
FL(p_t)=-alpha(1-p_t)^{gamma}ln(p_t)
Notice: y_pred is probability after softmax
gradient is d(Fl)/d(p_t) not d(Fl)/d(x) as described in paper
d(Fl)/d(p_t) * [p_t(1-p_t)] = d(Fl)/d(x)
Focal Loss for Dense Object Detection
https://arxiv.org/abs/1708.02002
Arguments:
y_true {tensor} -- ground truth labels, shape of [batch_size, num_cls]
y_pred {tensor} -- model's output, shape of [batch_size, num_cls]
Keyword Arguments:
gamma {float} -- (default: {2.0})
alpha {float} -- (default: {4.0})
Returns:
[tensor] -- loss.
"""
epsilon = 1.e-9
y_true = tf.convert_to_tensor(y_true, tf.float32)
y_pred = tf.convert_to_tensor(y_pred, tf.float32)
model_out = tf.add(y_pred, epsilon)
ce = tf.multiply(y_true, -tf.log(model_out))
weight = tf.multiply(y_true, tf.pow(tf.subtract(1., model_out), gamma))
fl = tf.multiply(alpha, tf.multiply(weight, ce))
reduced_fl = tf.reduce_max(fl, axis=1)
return tf.reduce_mean(reduced_fl)
return focal_loss_fixed
def per_class_accuracy(y_true, y_pred):
return tf.metrics.mean_per_class_accuracy(y_true, y_pred, num_classes=PARAMS['num_classes'])
def build_model(model_params,
optimizer,
loss,
METRICS):
if model_params['name']=='vgg16':
model_builder = vgg16.VGG16GrayScale(model_params)
elif model_params['name'].startswith('resnet'):
model_builder = resnet.ResNet(model_params)
base = model_builder.build_base()
model = model_builder.build_head(base)
model.compile(optimizer=optimizer,
loss=loss,
metrics=METRICS)
return model
def build_shallow(input_shape=(224,224,3),
num_classes=10,
optimizer=None,
loss=None,
METRICS=None):
model = tf.keras.models.Sequential()
model.add(layers.Conv2D(64, (7, 7), activation='relu', input_shape=input_shape, kernel_initializer=tf.initializers.GlorotNormal()))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (7, 7), activation='relu', kernel_initializer=tf.initializers.GlorotNormal()))
model.add(layers.MaxPooling2D((2, 2)))
model.add(layers.Conv2D(64, (7, 7), activation='relu', kernel_initializer=tf.initializers.GlorotNormal()))
model.add(layers.Flatten())
model.add(layers.Dense(64*2, activation='relu', kernel_initializer=tf.initializers.GlorotNormal()))
model.add(layers.Dense(num_classes,activation='softmax', kernel_initializer=tf.initializers.GlorotNormal()))
model.compile(optimizer=optimizer,
loss=loss,
metrics=METRICS)
return model
class ImageLogger:
'''Tensorflow 2.0 version'''
def __init__(self, log_dir: str, max_images: int, name: str):
self.file_writer = tf.summary.create_file_writer(log_dir)
self.log_dir = log_dir
self.max_images = max_images
self.name = name
self._counter = tf.Variable(0, dtype=tf.int64)
self.filepaths = []
def add_log(self, img, counter=None, name=None):
'''
Intention is to generalize this to an abstract class for logging to any experiment management platform (e.g. neptune, mlflow, etc)
Currently takes a filepath pointing to an image file and logs to current neptune experiment.
'''
# scaled_images = (img - tf.math.reduce_min(img))/(tf.math.reduce_max(img) - tf.math.reduce_min(img))
# keep = 0
# scaled_images = tf.image.convert_image_dtype(tf.squeeze(scaled_images[keep,:,:,:]), dtype=tf.uint8)
# scaled_images = tf.expand_dims(scaled_images, 0)
# tf.summary.image(name=self.name, data=scaled_images, step=self._counter, max_outputs=self.max_images)
scaled_img = (img - np.min(img))/(np.max(img) - np.min(img)) * 255.0
scaled_img = scaled_img.astype(np.uint32)
neptune.log_image(log_name= name or self.name,
x=counter,
y=scaled_img)
return scaled_img
def __call__(self, images, labels):
with self.file_writer.as_default():
scaled_images = (images - tf.math.reduce_min(images))/(tf.math.reduce_max(images) - tf.math.reduce_min(images))
keep = 0
scaled_images = tf.image.convert_image_dtype(tf.squeeze(scaled_images[keep,:,:,:]), dtype=tf.uint8)
scaled_images = tf.expand_dims(scaled_images, 0)
labels = tf.argmax(labels[[keep], :],axis=1)
tf.summary.image(name=self.name, data=scaled_images, step=self._counter, max_outputs=self.max_images)
filepath = os.path.join(self.log_dir,'sample_images',f'{self.name}-{self._counter}.jpg')
scaled_images = tf.image.encode_jpeg(tf.squeeze(scaled_images))
tf.io.write_file(filename=tf.constant(filepath),
contents=scaled_images)
# self.add_log(scaled_images)
self._counter.assign_add(1)
return images, labels
def _cond_apply(x, y, func, prob):
"""Conditionally apply func to x and y with probability prob.
Parameters
----------
x : type
Input to conditionally pass through func
y : type
Label
func : type
Function to conditionally be applied to x and y
prob : type
Probability of applying function, within range [0.0,1.0]
Returns
-------
x, y
"""
return tf.cond((tf.random.uniform([], 0, 1) >= (1.0 - prob)), lambda: func(x,y), lambda: (x,y))
class ImageAugmentor:
"""Short summary.
Parameters
----------
augmentations : dict
Maps a sequence of named augmentations to a scalar probability,
according to which they'll be conditionally applied in order.
resize_w_pad : tuple, default=None
Description of parameter `resize_w_pad`.
random_crop : tuple, default=None
Description of parameter `random_crop`.
random_jitter : dict
First applies resize_w_pad, then random_crop. If user desires only 1 of these, set this to None.
Should be a dict with 2 keys:
'resize':(height, width)
'crop_size':(crop_height,crop_width, channels)
Only 1 of these 3 kwargs should be provided to any given augmentor:
{'resize_w_pad', 'random_crop', 'random_jitter'}
Example values for each:
resize_w_pad=(224,224)
random_crop=(224,224,3)
random_jitter={'resize':(338,338),
'crop_size':(224,224, 3)}
seed : int, default=None
Random seed to apply to all augmentations
Examples
-------
Examples should be written in doctest format, and
should illustrate how to use the function/class.
>>>
Attributes
----------
augmentations
"""
def __init__(self,
name='',
augmentations={'rotate':1.0,
'flip':1.0,
'color':1.0,
'rgb2gray_3channel':1.0},
resize_w_pad=None,
random_crop=None,
random_jitter={'resize':(338,338),
'crop_size':(224,224,3)},
log_dir=None,
seed=None):
self.name = name
self.augmentations = augmentations
self.seed = seed
if resize_w_pad:
self.target_h = resize_w_pad[0]
self.target_w = resize_w_pad[1]
# self.resize = self.resize_w_pad
elif random_crop:
self.crop_size = random_crop
self.target_h = self.crop_size[0]
self.target_w = self.crop_size[1]
# self.resize = self.random_crop
elif random_jitter:
# self.target_h = tf.random.uniform([], random_jitter['crop_size'][0], random_jitter['resize'][0], dtype=tf.int32, seed=self.seed)
# self.target_w = tf.random.uniform([], random_jitter['crop_size'][1], random_jitter['resize'][1], dtype=tf.int32, seed=self.seed)
self.crop_size = random_jitter['crop_size']
# self.resize = self.random_jitter
self.target_h = random_jitter['crop_size'][0]
self.target_w = random_jitter['crop_size'][1]
self.resize = self.resize_w_pad
self.maps = {'rotate':self.rotate,
'flip':self.flip,
'color':self.color,
'rgb2gray_3channel':self.rgb2gray_3channel,
'rgb2gray_1channel':self.rgb2gray_1channel}
self.log_dir = log_dir
def rotate(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
"""Rotation augmentation
Args:
x, tf.Tensor: Image
label, tf.Tensor: arbitrary tensor, passes through unchanged
Returns:
Augmented image, label
"""
# Rotate 0, 90, 180, 270 degrees
return tf.image.rot90(x, tf.random.uniform(shape=[], minval=0, maxval=4, dtype=tf.int32,seed=self.seed)), label
def flip(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
"""Flip augmentation
Args:
x, tf.Tensor: Image to flip
label, tf.Tensor: arbitrary tensor, passes through unchanged
Returns:
Augmented image, label
"""
x = tf.image.random_flip_left_right(x, seed=self.seed)
x = tf.image.random_flip_up_down(x, seed=self.seed)
return x, label
def color(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
"""Color augmentation
Args:
x, tf.Tensor: Image
label, tf.Tensor: arbitrary tensor, passes through unchanged
Returns:
Augmented image, label
"""
x = tf.image.random_hue(x, 0.08, seed=self.seed)
x = tf.image.random_saturation(x, 0.6, 1.6, seed=self.seed)
x = tf.image.random_brightness(x, 0.05, seed=self.seed)
x = tf.image.random_contrast(x, 0.7, 1.3, seed=self.seed)
return x, label
def rgb2gray_3channel(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
"""Convert RGB image -> grayscale image, maintain number of channels = 3
Args:
x, tf.Tensor: Image
label, tf.Tensor: arbitrary tensor, passes through unchanged
Returns:
Augmented image, label
"""
x = tf.image.rgb_to_grayscale(x)
x = tf.image.grayscale_to_rgb(x)
return x, label
def rgb2gray_1channel(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
"""Convert RGB image -> grayscale image, reduce number of channels from 3 -> 1
Args:
x, tf.Tensor: Image
label, tf.Tensor: arbitrary tensor, passes through unchanged
Returns:
Augmented image, label
"""
x = tf.image.rgb_to_grayscale(x)
return x, label
def resize_w_pad(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
# TODO Finish this
# random_pad_image(x,min_image_size=None,max_image_size=None,pad_color=None,seed=self.seed)
return tf.image.resize_with_pad(x, target_height=self.target_h, target_width=self.target_w), label
def random_crop(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
return tf.image.random_crop(x, size=self.crop_size), label
@tf.function
def random_jitter(self, x: tf.Tensor, label: tf.Tensor) -> tf.Tensor:
x, label = self.resize_w_pad(x, label)
x, label = self.random_crop(x, label)
return x, label
def apply_augmentations(self, dataset: tf.data.Dataset):
"""
Call this function to apply all of the augmentation in the order of specification
provided to the constructor __init__() of ImageAugmentor.
Args:
dataset, tf.data.Dataset: must yield individual examples of form (x, y)
Returns:
Augmented dataset
"""
dataset = dataset.map(self.resize, num_parallel_calls=AUTOTUNE)
for aug_name, aug_p in self.augmentations.items():
aug = self.maps[aug_name]
dataset = dataset.map(lambda x,y: _cond_apply(x, y, aug, prob=aug_p), num_parallel_calls=AUTOTUNE)
# dataset = dataset.map(lambda x,y: _cond_apply(x, y, func=aug, prob=aug_p), num_parallel_calls=AUTOTUNE)
return dataset
class ImageLoggerCallback(Callback):
'''Tensorflow 2.0 version
Callback that keeps track of a tf.data.Dataset and logs the correct batch to neptune based on the current batch.
'''
def __init__(self, data :tf.data.Dataset, freq=1, max_images=-1, name='', encoder=None):
self.data = data
self.freq = freq
self.max_images = max_images
self.name = name
self.encoder=encoder
self.init_iterator()
def init_iterator(self):
self.data_iter = iter(self.data)
self._batch = 0
self._count = 0
self.finished = False
def yield_batch(self):
batch_data = next(self.data_iter)
self._batch += 1
self._count += batch_data[0].shape[0]
return batch_data
def add_log(self, img, counter=None, name=None):
'''
Intention is to generalize this to an abstract class for logging to any experiment management platform (e.g. neptune, mlflow, etc)
Currently takes a filepath pointing to an image file and logs to current neptune experiment.
'''
scaled_img = (img - np.min(img))/(np.max(img) - np.min(img)) * 255.0
scaled_img = scaled_img.astype(np.uint32)
neptune.log_image(log_name= name or self.name,
x=counter,
y=scaled_img)
return scaled_img
def on_train_batch_begin(self, batch, logs=None):
if batch % self.freq or self.finished:
return
while batch >= self._batch:
x, y = self.yield_batch()
if self.max_images==-1:
self.max_images=x.shape[0]
if x.ndim==3:
np.newaxis(x, axis=0)
if x.shape[0]>self.max_images:
x = x[:self.max_images,...]
y = y[:self.max_images,...]
x = x.numpy()
y = np.argmax(y.numpy(),axis=1)
if self.encoder:
y = self.encoder.decode(y)
for i in range(x.shape[0]):
# self.add_log(x[i,...], counter=i, name = f'{self.name}-{y[i]}-batch_{str(self._batch).zfill(3)}')
self.add_log(x[i,...], counter=self._count+i, name = f'{self.name}-{y[i]}')
print(f'Batch {self._batch}: Logged {np.max([x.shape[0],self.max_images])} {self.name} images to neptune')
def on_epoch_end(self, epoch, logs={}):
self.finished = True
class ConfusionMatrixCallback(Callback):
'''Tensorflow 2.0 version'''
def __init__(self, log_dir, imgs : dict, labels : dict, classes, freq=1, include_train=False, seed=None):
self.file_writer = tf.summary.create_file_writer(log_dir)
self.log_dir = log_dir
self.seed = seed
self._counter = 0
assert np.all(np.array(imgs.keys()) == np.array(labels.keys()))
self.imgs = imgs
for k,v in labels.items():
if v.ndim==2:
labels[k] = tf.argmax(v,axis=-1)
self.labels = labels
self.num_samples = {k:l.numpy().shape[0] for k,l in labels.items()}
self.classes = classes
self.freq = freq
self.include_train = include_train
def log_confusion_matrix(self, model, imgs, labels, epoch, name='', norm_cm=False):
pred_labels = model.predict_classes(imgs)
# pred_labels = tf.argmax(pred_labels,axis=-1)
pred_labels = pred_labels[:,None]
con_mat = tf.math.confusion_matrix(labels=labels, predictions=pred_labels, num_classes=len(self.classes)).numpy()
if norm_cm:
con_mat = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)
con_mat_df = pd.DataFrame(con_mat,
index = self.classes,
columns = self.classes)
figure = plt.figure(figsize=(12, 12))
sns.heatmap(con_mat_df, annot=True, cmap=plt.cm.Blues)
plt.tight_layout()
plt.ylabel('True label')
plt.xlabel('Predicted label')
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
image = tf.image.decode_png(buf.getvalue(), channels=4)
image = tf.expand_dims(image, 0)
with self.file_writer.as_default():
tf.summary.image(name=name+'_confusion_matrix', data=image, step=self._counter)
neptune.log_image(log_name=name+'_confusion_matrix',
x=self._counter,
y=figure)
plt.close(figure)
self._counter += 1
return image
def on_epoch_end(self, epoch, logs={}):
if (not self.freq) or (epoch%self.freq != 0):
return
if self.include_train:
cm_summary_image = self.log_confusion_matrix(self.model, self.imgs['train'], self.labels['train'], epoch=epoch, name='train')
cm_summary_image = self.log_confusion_matrix(self.model, self.imgs['val'], self.labels['val'], epoch=epoch, name='val')
####################################################################################
####################################################################################
####################################################################################
neptune.init(project_qualified_name=args.neptune_project_name)
# neptune_tb.integrate_with_tensorflow()
experiment_dir = '/media/data/jacob/sandbox_logs'
experiment_name = args.experiment_name
experiment_start_time = arrow.utcnow().format('YYYY-MM-DD_HH-mm-ss')
log_dir =os.path.join(experiment_dir, experiment_name, 'log_dir',PARAMS['loss'], experiment_start_time)
ensure_dir_exists(log_dir)
print('Tensorboard log_dir: ', log_dir)
# os.system(f'neptune tensorboard {log_dir} --project {args.neptune_project_name}')
weights_best = os.path.join(log_dir, 'model_ckpt.h5')
restore_best_weights=False
histogram_freq=0
patience=25
num_epochs = PARAMS['num_epochs']
initial_epoch=0
src_db = pyleaves.DATABASE_PATH
datasets = {
'PNAS': pnas_dataset.PNASDataset(src_db=src_db),
'Leaves': leaves_dataset.LeavesDataset(src_db=src_db),
'Fossil': fossil_dataset.FossilDataset(src_db=src_db)
}
# data = datasets[PARAMS['dataset_name']]
data_config = stuf(threshold=PARAMS['data_threshold'],
num_classes=PARAMS['num_classes'] ,
data_splits_meta={
'train':PARAMS['train_size'],
'val':PARAMS['val_size'],
'test':PARAMS['test_size']
}
)
preprocess_input = get_preprocessing_func(PARAMS['model_name'])
preprocess_input(tf.zeros([4, 224, 224, 3]))
load_example = partial(_load_uint8_example, num_classes=data_config.num_classes)
# load_example = partial(_load_example, num_classes=data_config.num_classes)
if PARAMS['num_channels']==3:
color_aug = {'rgb2gray_3channel':1.0}
elif PARAMS['num_channels']==1:
color_aug = {'rgb2gray_1channel':1.0}
resize_w_pad=None
random_jitter=None
if not PARAMS['random_jitter']['resize']:
resize_w_pad = PARAMS['image_size']
else:
random_jitter=PARAMS['random_jitter']
TRAIN_image_augmentor = ImageAugmentor(name='train',
augmentations={**PARAMS["augmentations"],
**color_aug},#'rotate':1.0,'flip':1.0,**color_aug},
resize_w_pad=resize_w_pad,
random_crop=None,
random_jitter=random_jitter,
log_dir=log_dir,
seed=None)
VAL_image_augmentor = ImageAugmentor(name='val',
augmentations={**color_aug},
resize_w_pad=PARAMS['image_size'],
random_crop=None,
random_jitter=None,
log_dir=log_dir,
seed=None)
TEST_image_augmentor = ImageAugmentor(name='test',
augmentations={**color_aug},
resize_w_pad=PARAMS['image_size'],
random_crop=None,
random_jitter=None,
log_dir=log_dir,
seed=None)
def neptune_log_augmented_images(split_data, num_demo_samples=40, PARAMS=PARAMS):
num_demo_samples = 40
cm_data_x = {'train':[],'val':[]}
cm_data_y = {'train':[],'val':[]}
cm_data_x['train'], cm_data_y['train'] = next(iter(get_data_loader(data=split_data['train'], data_subset_mode='train', batch_size=num_demo_samples, infinite=True, augment=False,seed=2836)))
cm_data_x['val'], cm_data_y['val'] = next(iter(get_data_loader(data=split_data['val'], data_subset_mode='val', batch_size=num_demo_samples, infinite=True, augment=False, seed=2836)))
for (k_x,v_x), (k_y, v_y) in zip(cm_data_x.items(), cm_data_y.items()):
x = tf.data.Dataset.from_tensor_slices(v_x)
y = tf.data.Dataset.from_tensor_slices(v_y)
xy_data = tf.data.Dataset.zip((x, y))
v = xy_data.map(VAL_image_augmentor.resize, num_parallel_calls=AUTOTUNE)
v_aug = TRAIN_image_augmentor.apply_augmentations(xy_data)
v_x, v_y = [i.numpy() for i in next(iter(v.batch(10*num_demo_samples)))]
v_x_aug, v_y_aug = [i.numpy() for i in next(iter(v_aug.batch(10*num_demo_samples)))]
k = k_x
for i in range(num_demo_samples):
print(f'Neptune: logging {k}_{i}')
print(f'{v_x[i].shape}, {v_x_aug[i].shape}')
idx = np.random.randint(0,len(v_x))
if True: #'train' in k:
TRAIN_image_augmentor.logger.add_log(v_x[idx],counter=i, name=k)
TRAIN_image_augmentor.logger.add_log(v_x_aug[idx],counter=i, name=k+'_aug')
def get_data_loader(data : tuple, data_subset_mode='train', batch_size=32, num_classes=None, infinite=True, augment=True, seed=2836):
num_samples = len(data[0])
x = tf.data.Dataset.from_tensor_slices(data[0])
labels = tf.data.Dataset.from_tensor_slices(data[1])
data = tf.data.Dataset.zip((x, labels))
data = data.cache()
if data_subset_mode == 'train':
data = data.shuffle(buffer_size=num_samples)
# data = data.map(lambda x,y: (tf.image.convert_image_dtype(load_img(x)*255.0,dtype=tf.uint8),y), num_parallel_calls=-1)
# data = data.map(load_example, num_parallel_calls=AUTOTUNE)
data = data.map(load_example, num_parallel_calls=AUTOTUNE)
data = data.map(lambda x,y: (preprocess_input(x), y), num_parallel_calls=AUTOTUNE)
if infinite:
data = data.repeat()
if data_subset_mode == 'train':
data = data.shuffle(buffer_size=200, seed=seed)
augmentor = TRAIN_image_augmentor
elif data_subset_mode == 'val':
augmentor = VAL_image_augmentor
elif data_subset_mode == 'test':
augmentor = TEST_image_augmentor
if augment:
data = augmentor.apply_augmentations(data)
data = data.batch(batch_size, drop_remainder=True)
return data.prefetch(AUTOTUNE)
def get_tfds_data_loader(data : tf.data.Dataset, data_subset_mode='train', batch_size=32, num_samples=100, num_classes=19, infinite=True, augment=True, seed=2836):
def encode_example(x, y):
x = tf.image.convert_image_dtype(x, tf.float32) * 255.0
y = _encode_label(y, num_classes=num_classes)
return x, y
test_d = next(iter(data))
print(test_d[0].numpy().min())
print(test_d[0].numpy().max())
data = data.shuffle(buffer_size=num_samples) \
.cache() \
.map(encode_example, num_parallel_calls=AUTOTUNE)
test_d = next(iter(data))
print(test_d[0].numpy().min())
print(test_d[0].numpy().max())
data = data.map(preprocess_input, num_parallel_calls=AUTOTUNE)
test_d = next(iter(data))
print(test_d[0].numpy().min())
print(test_d[0].numpy().max())
if data_subset_mode == 'train':
data = data.shuffle(buffer_size=100, seed=seed)
augmentor = TRAIN_image_augmentor
elif data_subset_mode == 'val':
augmentor = VAL_image_augmentor
elif data_subset_mode == 'test':
augmentor = TEST_image_augmentor
if augment:
data = augmentor.apply_augmentations(data)
test_d = next(iter(data))
print(test_d[0].numpy().min())
print(test_d[0].numpy().max())
data = data.batch(batch_size, drop_remainder=True)
if infinite:
data = data.repeat()
return data.prefetch(AUTOTUNE)
# y_true = [[0, 1, 0], [0, 0, 1]]
# y_pred = [[0.05, 0.95, 0], [0.1, 0.8, 0.1]]
def accuracy(y_true, y_pred):
y_pred = tf.argmax(y_pred, axis=-1)
y_true = tf.argmax(y_true, axis=-1)
return tf.reduce_mean(tf.cast(tf.equal(y_true, y_pred), tf.float32))
def true_pos(y_true, y_pred):
# y_true = K.ones_like(y_true)
return K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
def false_pos(y_true, y_pred):
# y_true = K.ones_like(y_true)
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
all_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
return all_positives - true_positives
def true_neg(y_true, y_pred):
# y_true = K.ones_like(y_true)
return K.sum(1-K.round(K.clip(y_true * y_pred, 0, 1)))
def recall(y_true, y_pred):
# y_true = K.ones_like(y_true)
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
all_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (all_positives + K.epsilon())
return recall
def precision(y_true, y_pred):
y_true = K.ones_like(y_true)
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
# tf.print(y_true, y_pred)
return precision
def f1_score(y_true, y_pred):
m_precision = precision(y_true, y_pred)
m_recall = recall(y_true, y_pred)
# pdb.set_trace()
return 2*((m_precision*m_recall)/(m_precision+m_recall+K.epsilon()))
# def false_neg(y_true, y_pred):
# y_true = K.ones_like(~y_true)
# true_neg = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
# all_negative = K.sum(K.round(K.clip(y_true, 0, 1)))
# return all_negatives - true_
# return K.mean(K.argmax(y_true,axis=1)*K.argmax(y_pred,axis=1))
# 'accuracy',
# metrics.TrueNegatives(name='tn'),
# metrics.FalseNegatives(name='fn'),
METRICS = [
f1_score,
metrics.TruePositives(name='tp'),
metrics.FalsePositives(name='fp'),
metrics.CategoricalAccuracy(name='accuracy'),
metrics.TopKCategoricalAccuracy(name='top_3_categorical_accuracy', k=3),
metrics.TopKCategoricalAccuracy(name='top_5_categorical_accuracy', k=5)
]
PARAMS['sys.argv'] = ' '.join(sys.argv)
with neptune.create_experiment(name=experiment_name, params=PARAMS, upload_source_files=[__file__]):
print('Logging experiment tags:')
for tag in args.tags:
print(tag)
neptune.append_tag(tag)
neptune.append_tag(PARAMS['dataset_name'])
neptune.append_tag(PARAMS['model_name'])
neptune.log_artifact(args.config_path)
cm_data_x = {'train':[],'val':[]}
cm_data_y = {'train':[],'val':[]}
if PARAMS['dataset_name'] in tfds.list_builders():
num_demo_samples=40
tfds_builder = tfds.builder(PARAMS['dataset_name'])
tfds_builder.download_and_prepare()
num_samples = tfds_builder.info.splits['train'].num_examples
num_samples_dict = {'train':int(num_samples*PARAMS['train_size']),
'val':int(num_samples*PARAMS['val_size']),
'test':int(num_samples*PARAMS['test_size'])}
classes = tfds_builder.info.features['label'].names
num_classes = len(classes)
train_slice = [0,int(PARAMS['train_size']*100)]
val_slice = [int(PARAMS['train_size']*100), int((PARAMS['train_size']+PARAMS['val_size'])*100)]
test_slice = [100 - int(PARAMS['test_size']*100), 100]
tfds_train_data = tfds.load(PARAMS['dataset_name'], split=f"train[{train_slice[0]}%:{train_slice[1]}%]", shuffle_files=True, as_supervised=True)
tfds_validation_data = tfds.load(PARAMS['dataset_name'], split=f"train[{val_slice[0]}%:{val_slice[1]}%]", shuffle_files=True, as_supervised=True)
tfds_test_data = tfds.load(PARAMS['dataset_name'], split=f"train[{test_slice[0]}%:{test_slice[1]}%]", shuffle_files=True, as_supervised=True)
# PARAMS['batch_size']=1
train_data = get_tfds_data_loader(data = tfds_train_data, data_subset_mode='train', batch_size=PARAMS['batch_size'], num_samples=num_samples_dict['train'], num_classes=num_classes, infinite=True, augment=True, seed=2836)
validation_data = get_tfds_data_loader(data = tfds_validation_data, data_subset_mode='val', batch_size=PARAMS['batch_size'], num_samples=num_samples_dict['val'], num_classes=num_classes, infinite=True, augment=True, seed=2837)
test_data = get_tfds_data_loader(data = tfds_test_data, data_subset_mode='test', batch_size=PARAMS['batch_size'], num_samples=num_samples_dict['test'], num_classes=num_classes, infinite=True, augment=True, seed=2838)
# tfds_train_data = tfds.load(PARAMS['dataset_name'], split=f"train[{train_slice[0]}%:{train_slice[1]}%]", shuffle_files=True, as_supervised=True)
# tfds_validation_data = tfds.load(PARAMS['dataset_name'], split=f"train[{val_slice[0]}%:{val_slice[1]}%]", shuffle_files=True, as_supervised=True)
# tfds_test_data = tfds.load(PARAMS['dataset_name'], split=f"train[{test_slice[0]}%:{test_slice[1]}%]", shuffle_files=True, as_supervised=True)
split_data = {'train':get_tfds_data_loader(data = tfds_train_data, data_subset_mode='train', batch_size=num_demo_samples, num_samples=num_samples_dict['train'], num_classes=num_classes, infinite=True, augment=True, seed=2836),
'val':get_tfds_data_loader(data = tfds_validation_data, data_subset_mode='val', batch_size=num_demo_samples, num_samples=num_samples_dict['val'], num_classes=num_classes, infinite=True, augment=True, seed=2837),
'test':get_tfds_data_loader(data = tfds_test_data, data_subset_mode='test', batch_size=num_demo_samples, num_samples=num_samples_dict['test'], num_classes=num_classes, infinite=True, augment=True, seed=2838)
}
steps_per_epoch=num_samples_dict['train']//PARAMS['batch_size']
validation_steps=num_samples_dict['val']//PARAMS['batch_size']
cm_data_x['train'], cm_data_y['train'] = next(iter(split_data['train']))
cm_data_x['val'], cm_data_y['val'] = next(iter(split_data['val']))
else:
data = datasets[PARAMS['dataset_name']]
neptune.set_property('num_classes',data.num_classes)
neptune.set_property('class_distribution',data.metadata.class_distribution)
encoder = base_dataset.LabelEncoder(data.data.family)
split_data = base_dataset.preprocess_data(data, encoder, data_config)
# import pdb;pdb.set_trace()
for subset, subset_data in split_data.items():
split_data[subset] = [list(i) for i in unzip(subset_data)]
PARAMS['batch_size'] = 32
steps_per_epoch=len(split_data['train'][0])//PARAMS['batch_size']#//10
validation_steps=len(split_data['val'][0])//PARAMS['batch_size']#//10
split_datasets = {
k:base_dataset.BaseDataset.from_dataframe(
pd.DataFrame({
'path':v[0],
'family':v[1]
})) \
for k,v in split_data.items()
}
for k,v in split_datasets.items():
print(k, v.num_classes)
classes = split_datasets['train'].classes
train_data=get_data_loader(data=split_data['train'], data_subset_mode='train', batch_size=PARAMS['batch_size'], infinite=True, augment=True, seed=2836)
validation_data=get_data_loader(data=split_data['val'], data_subset_mode='val', batch_size=PARAMS['batch_size'], infinite=True, augment=True, seed=2837)
if 'test' in split_data.keys():
test_data=get_data_loader(data=split_data['test'], data_subset_mode='test', batch_size=PARAMS['batch_size'], infinite=True, augment=True, seed=2838)
num_demo_samples=150
# neptune_log_augmented_images(split_data, num_demo_samples=num_demo_samples, PARAMS=PARAMS)
cm_data_x['train'], cm_data_y['train'] = next(iter(get_data_loader(data=split_data['train'], data_subset_mode='train', batch_size=num_demo_samples, infinite=True, augment=True, seed=2836)))
cm_data_x['val'], cm_data_y['val'] = next(iter(get_data_loader(data=split_data['val'], data_subset_mode='val', batch_size=num_demo_samples, infinite=True, augment=True, seed=2836)))
########################################################################################
train_image_logger_cb = ImageLoggerCallback(data=train_data, freq=20, max_images=-1, name='train', encoder=encoder)
val_image_logger_cb = ImageLoggerCallback(data=validation_data, freq=20, max_images=-1, name='val', encoder=encoder)
########################################################################################
cm_callback = ConfusionMatrixCallback(log_dir, cm_data_x, cm_data_y, classes=classes, seed=PARAMS['seed'], include_train=True)
checkpoint = ModelCheckpoint(weights_best, monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=False, mode='min',restore_best_weights=restore_best_weights)
tfboard = TensorBoard(log_dir=log_dir, histogram_freq=histogram_freq, write_images=True)
early = EarlyStopping(monitor='val_loss', patience=patience, verbose=1)
callbacks = [checkpoint,tfboard,early, cm_callback, neptune_logger, train_image_logger_cb, val_image_logger_cb]
##########################
if PARAMS['optimizer'] == 'Adam':
optimizer = tf.keras.optimizers.Adam(
learning_rate=PARAMS['lr']
)
elif PARAMS['optimizer'] == 'Nadam':
optimizer = tf.keras.optimizers.Nadam(
learning_rate=PARAMS['lr']
)
elif PARAMS['optimizer'] == 'SGD':
optimizer = tf.keras.optimizers.SGD(
learning_rate=PARAMS['lr']
)
##########################
if PARAMS['loss']=='focal_loss':
loss = focal_loss(gamma=2.0, alpha=4.0)
elif PARAMS['loss']=='categorical_crossentropy':
loss = 'categorical_crossentropy'
##########################
model_params = stuf(name=PARAMS['model_name'],
model_dir=os.path.join(experiment_dir, experiment_name, 'models'),
num_classes=PARAMS['num_classes'],
frozen_layers = PARAMS['frozen_layers'],
input_shape = (*PARAMS['image_size'],PARAMS['num_channels']),
base_learning_rate = PARAMS['lr'],
regularization = PARAMS['regularization'])
####
if PARAMS['model_name']=='shallow':
model = build_shallow(input_shape=model_params.input_shape,
num_classes=PARAMS['num_classes'],
optimizer=optimizer,
loss=loss,
METRICS=METRICS)
else:
model = build_model(model_params,
optimizer,
loss,
METRICS)
print(f"TRAINING {PARAMS['model_name']}")
model.summary(print_fn=lambda x: neptune.log_text('model_summary', x))
history = model.fit(train_data,
epochs=num_epochs,
callbacks=callbacks,
validation_data=validation_data,
shuffle=True,
initial_epoch=initial_epoch,
steps_per_epoch=steps_per_epoch,
validation_steps=validation_steps)
if 'test' in split_data:
results = model.evaluate(test_data,
steps=len(split_data['test'][0]))
else:
results = model.evaluate(validation_data,
steps=validation_steps)