def resynthesis_model(testing_dataset,
_run,
_log,
ours=True,
validation=False):
# added import inside the function to prevent conflicts if this method is not being tested
sys.path.insert(
0,
os.path.join(os.getcwd(), os.path.dirname(os.path.dirname(__file__)),
'driving_uncertainty'))
from driving_uncertainty.test_fishy_torch import AnomalyDetector
detector = AnomalyDetector(ours=ours)
fsdata = FSData(**testing_dataset)
# Hacks because tf.data is shit and we need to translate the dict keys
def data_generator():
dataset = fsdata.validation_set if validation else fsdata.testset
for item in dataset:
data = fsdata._get_data(training_format=False, **item)
out = {}
for m in fsdata.modalities:
blob = crop_multiple(data[m])
if m == 'rgb':
m = 'image_left'
if 'mask' not in fsdata.modalities and m == 'labels':
m = 'mask'
out[m] = blob
yield out
data_types = {}
for key, item in fsdata.get_data_description()[0].items():
if key == 'rgb':
key = 'image_left'
if 'mask' not in fsdata.modalities and key == 'labels':
key = 'mask'
data_types[key] = item
data = tf.data.Dataset.from_generator(data_generator, data_types)
fs = bdlb.load(benchmark="fishyscapes", download_and_prepare=False)
if ours:
model_id = 'SynBoost'
else:
model_id = 'Resynthesis'
def wrapper(image):
image = image.numpy().astype('uint8')
ret = detector.estimator_worker(image)
return ret
_run.info['{}_anomaly'.format(model_id)] = fs.evaluate(wrapper, data)
def entropy_maximization(testing_dataset, _run, _log, validation=False):
# added import inside the function to prevent conflicts if this method is not being tested
sys.path.insert(
0,
os.path.join(os.getcwd(), os.path.dirname(os.path.dirname(__file__)),
'robin', 'entropy_maximization'))
from entropy_maximization.foward_pass import init_model, get_softmax_entropy
# Disable all GPUS for tensorflow
tf.config.experimental.set_visible_devices([], 'GPU')
fsdata = FSData(**testing_dataset)
# Hacks because tf.data is shit and we need to translate the dict keys
def data_generator():
dataset = fsdata.validation_set if validation else fsdata.testset
for item in dataset:
data = fsdata._get_data(training_format=False, **item)
out = {}
for m in fsdata.modalities:
blob = crop_multiple(data[m])
if m == 'rgb':
m = 'image_left'
if 'mask' not in fsdata.modalities and m == 'labels':
m = 'mask'
out[m] = blob
yield out
data_types = {}
for key, item in fsdata.get_data_description()[0].items():
if key == 'rgb':
key = 'image_left'
if 'mask' not in fsdata.modalities and key == 'labels':
key = 'mask'
data_types[key] = item
data = tf.data.Dataset.from_generator(data_generator, data_types)
fs = bdlb.load(benchmark="fishyscapes", download_and_prepare=False)
class ForwardPass(object):
def __init__(self):
self.model = init_model()
def compute_entropy(self, image):
image = image.numpy().astype('uint8')
softmax_entropy = get_softmax_entropy(self.model, image)
anomaly_score = tf.convert_to_tensor(softmax_entropy,
dtype=tf.float32)
return anomaly_score
get_anomaly_score = ForwardPass().compute_entropy
_run.info['entropy_max_anomaly'] = fs.evaluate(get_anomaly_score, data)
def resynthesis_model(image_path,
name,
ours,
labels=None,
testing_dataset=None,
batching=None,
validation=None):
# added import inside the function to prevent conflicts if this method is not being tested
sys.path.insert(
0,
os.path.join(os.getcwd(), os.path.dirname(os.path.dirname(__file__)),
'driving_uncertainty'))
from driving_uncertainty.test_fishy_torch import AnomalyDetector
detector = AnomalyDetector(ours=ours)
image = cv2.imread(image_path).astype('uint8')
out = detector.estimator_worker(image).numpy()
min_val, max_val = out.min(), out.max()
disp = (out - min_val) / (max_val - min_val)
disp = 255 - (np.clip(disp, 0, 1) * 255).astype('uint8')
directory, filename = os.path.split(image_path)
if labels is None:
cv2.imwrite(
os.path.join(directory, f'{filename.split(".")[0]}_{name}.jpg'),
disp)
return
# since we have labels, check for the accuracy
def data_generator():
rgb = cv2.imread(image_path).astype('float32')
label = cv2.imread(labels, cv2.IMREAD_ANYDEPTH).astype('int32')
yield {'image_left': rgb, 'mask': label}
data = tf.data.Dataset.from_generator(data_generator, {
'image_left': tf.float32,
'mask': tf.int32
})
def eval_func(image):
image = image.numpy().astype('uint8')
ret = detector.estimator_worker(image)
return ret
fs = bdlb.load(benchmark='fishyscapes', download_and_prepare=False)
metrics = fs.evaluate(eval_func, data)
print(metrics['AP'], flush=True)
cv2.imwrite(
os.path.join(
directory,
f'{filename.split(".")[0]}_{name}_AP{100 * metrics["AP"]:.2f}.jpg'
), disp)
def saved_model(testing_dataset,
model_id,
_run,
_log,
batching=False,
validation=False):
fsdata = FSData(**testing_dataset)
# Hacks because tf.data is shit and we need to translate the dict keys
def data_generator():
dataset = fsdata.validation_set if validation else fsdata.testset
for item in dataset:
data = fsdata._get_data(training_format=False, **item)
out = {}
for m in fsdata.modalities:
blob = crop_multiple(data[m])
if m == 'rgb':
m = 'image_left'
if 'mask' not in fsdata.modalities and m == 'labels':
m = 'mask'
out[m] = blob
yield out
data_types = {}
for key, item in fsdata.get_data_description()[0].items():
if key == 'rgb':
key = 'image_left'
if 'mask' not in fsdata.modalities and key == 'labels':
key = 'mask'
data_types[key] = item
data = tf.data.Dataset.from_generator(data_generator, data_types)
ZipFile(load_gdrive_file(model_id,
'zip')).extractall('/tmp/extracted_module')
tf.compat.v1.enable_resource_variables()
net = tf.saved_model.load('/tmp/extracted_module')
def eval_func(image):
if batching:
image = tf.expand_dims(image, 0)
out = net.signatures['serving_default'](tf.cast(image, tf.float32))
for key, val in out.items():
print(key, val.shape, flush=True)
return out['anomaly_score']
fs = bdlb.load(benchmark="fishyscapes", download_and_prepare=False)
_run.info['{}_anomaly'.format(model_id)] = fs.evaluate(eval_func, data)
def __getitem__(self, index):
"""__getitem__
:param index:
"""
import bdlb
fs = bdlb.load(benchmark="fishyscapes")
# automatically downloads the dataset
data = fs.get_dataset()
img = []
lbl = []
for i, blob in enumerate(data.take(10)):
img.append(blob['image_left'])
lbl.append(blob['mask'])
print(blob['image_left'])
return img, lbl
def main(argv):
print(argv)
print(FLAGS)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
out_dir = os.path.join(FLAGS.output_dir, 'EnsembleMCdropout', current_time)
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3),
realworld=(512, 512, 3))[FLAGS.level]
hparams = dict(dropout_rate=FLAGS.dropout_rate,
num_base_filters=FLAGS.num_base_filters,
learning_rate=FLAGS.learning_rate,
l2_reg=FLAGS.l2_reg,
input_shape=input_shape)
classifiers = list()
for checkpoint in FLAGS.model_checkpoints:
classifier = VGGDrop(**hparams)
classifier.load_weights(checkpoint)
classifier.summary()
classifiers.append(classifier)
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
_, _, ds_test = dtask.datasets
##############
# Evaluation #
##############
dtask.evaluate(functools.partial(predict,
models=classifiers,
num_samples=FLAGS.num_mc_samples,
type=FLAGS.uncertainty),
dataset=ds_test,
output_dir=os.path.join(out_dir, 'evaluation'))
def ood_segmentation(testing_dataset, _run, _log, ours=True, validation=False):
# added import inside the function to prevent conflicts if this method is not being tested
sys.path.insert(
0,
os.path.join(os.getcwd(), os.path.dirname(os.path.dirname(__file__)),
'awesomemang', 'ood_segmentation'))
from ood_segmentation import network_manager
ood_detector = network_manager.get_net()
fsdata = FSData(**testing_dataset)
# Hacks because tf.data is shit and we need to translate the dict keys
def data_generator():
dataset = fsdata.validation_set if validation else fsdata.testset
for item in dataset:
data = fsdata._get_data(training_format=False, **item)
out = {}
for m in fsdata.modalities:
blob = crop_multiple(data[m])
if m == 'rgb':
m = 'image_left'
if 'mask' not in fsdata.modalities and m == 'labels':
m = 'mask'
out[m] = blob
yield out
data_types = {}
for key, item in fsdata.get_data_description()[0].items():
if key == 'rgb':
key = 'image_left'
if 'mask' not in fsdata.modalities and key == 'labels':
key = 'mask'
data_types[key] = item
data = tf.data.Dataset.from_generator(data_generator, data_types)
fs = bdlb.load(benchmark="fishyscapes", download_and_prepare=False)
def wrapper(image):
image = image.numpy().astype('uint8')
main_out, anomaly_score = ood_detector(image, preprocess=True)
return anomaly_score
_run.info['awesomemango_anomaly2'] = fs.evaluate(wrapper, data)
def main(argv):
print(argv)
print(FLAGS)
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3),
realworld=(512, 512, 3))[FLAGS.level]
hparams = dict(num_base_filters=FLAGS.num_base_filters,
learning_rate=FLAGS.learning_rate,
input_shape=input_shape)
classifier = VGGFlipout(**hparams)
classifier.summary()
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
ds_train, ds_validation, ds_test = dtask.datasets
#################
# Training Loop #
#################
history = classifier.fit(
ds_train,
epochs=FLAGS.num_epochs,
validation_data=ds_validation,
class_weight=dtask.class_weight(),
callbacks=[
tfk.callbacks.TensorBoard(
log_dir=os.path.join(FLAGS.output_dir, "tensorboard"),
update_freq="epoch",
write_graph=True,
histogram_freq=1,
),
tfk.callbacks.ModelCheckpoint(
filepath=os.path.join(
FLAGS.output_dir,
"checkpoints",
"weights-{epoch}.ckpt",
),
verbose=1,
save_weights_only=True,
)
],
)
plotting.tfk_history(history,
output_dir=os.path.join(FLAGS.output_dir, "history"))
##############
# Evaluation #
##############
dtask.evaluate(functools.partial(predict,
model=classifier,
num_samples=FLAGS.num_mc_samples,
type=FLAGS.uncertainty),
dataset=ds_test,
output_dir=FLAGS.output_dir)
def main(argv):
print(argv)
print(FLAGS)
#
# Logging / tensorboard
#
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
out_dir = os.path.join(FLAGS.output_dir, 'EDL', current_time)
file_writer = tf.summary.create_file_writer(os.path.join(out_dir, "tensorboard"))
file_writer.set_as_default()
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
ds_train, ds_validation, ds_test = dtask.datasets
assert isinstance(ds_train, tf.data.Dataset)
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3), realworld=(512, 512, 3))[FLAGS.level]
global_step = tf.Variable(initial_value=0, name="global_step", trainable=False, dtype=tf.int64)
tf.summary.experimental.set_step(global_step)
logits_model = model.VGG_model(dropout_rate=FLAGS.dropout_rate,
num_base_filters=FLAGS.num_base_filters,
l2_reg=FLAGS.l2_reg,
input_shape=input_shape,
output_dim=2)
print('********** Output dir: {} ************'.format(out_dir))
# Create optimiser
optimizer = tfk.optimizers.Adam(FLAGS.learning_rate)
#################
# Training Loop #
#################
#
# keep results for plotting
train_loss_results = []
train_accuracy_results = []
test_accuracy_results = []
#
# Keep track of statistics
train_loss_avg = tfk.metrics.Mean()
train_entropy_avg = tfk.metrics.Mean()
train_accuracy = tfk.metrics.Accuracy()
test_accuracy = tfk.metrics.Accuracy()
#
# Set up checkpointing
checkpoint_dir = os.path.join(out_dir, 'checkpoints')
checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
root = tf.train.Checkpoint(optimizer=optimizer, model=logits_model, optimizer_step=global_step)
#
for epoch in range(FLAGS.num_epochs):
#
# Reset statistics
train_loss_avg.reset_states()
train_entropy_avg.reset_states()
train_accuracy.reset_states()
test_accuracy.reset_states()
#
# Calculate annealing coefficient
lambda_t = model.annealing_coefficient(epoch)
#
# Make summaries
tf.summary.scalar('lambda_t', lambda_t)
# tf.summary.scalar('global_step', global_step)
#
# Training loop: for each batch
for batch, (x, y) in enumerate(ds_train):
# print('x: ', x.shape)
# print('y: ', y.shape)
y_one_hot = tf.one_hot(y, depth=2, name="y_one_hot") # Make one-hot for MSE loss
# print('y one hot: ', y_one_hot.shape)
#
# Calculate the loss
with tf.GradientTape() as tape:
#
# Calculate the logits and the evidence
logits = logits_model(x)
evidence, alpha, alpha0, p_mean, p_pos = model.EDL_model(logits)
#
# Calculate the loss
alpha_mod = (1 - y_one_hot) * evidence + 1
regularisation_term = lambda_t * model.loss_regulariser(alpha_mod)
mse_term = model.EDL_loss()(y_one_hot, alpha, epoch)
#mse_term = model.mse_loss(y_one_hot, alpha, epoch)
loss = weight_loss(y_one_hot,mse_term, dtask.class_weight())
#loss = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_one_hot)
#probs = alpha/tf.reduce_sum(alpha,axis=1,keepdims=True)
#loss = tf.reduce_mean(-tf.reduce_sum(y_one_hot * tf.math.log(probs+1e-30), axis=1))
# loss = mse_term
# print('Loss: ', loss.shape)
#
# Calculate gradients
grads = tape.gradient(loss, logits_model.trainable_variables)
#
# Apply gradients
optimizer.apply_gradients(zip(grads, logits_model.trainable_variables))
#
# Calculate the expected entropy
exp_entropy = model.tf_dirichlet_expected_entropy(alpha)
#
# compare predicted label to actual label
prediction = tf.argmax(logits, axis=1, output_type=y.dtype)
#
# Make summaries
tf.summary.histogram('alpha', alpha)
tf.summary.histogram('expected_entropy', exp_entropy)
tf.summary.histogram('regularisation', regularisation_term)
tf.summary.histogram('mse', mse_term)
tf.summary.histogram('loss', loss)
#
# Track progress
train_accuracy.update_state(prediction, y)
train_entropy_avg.update_state(exp_entropy)
train_loss_avg.update_state(loss) # add current batch loss
#
# Evaluate on test accuracy
for test_batch, (x, y) in enumerate(ds_test.take(10)):
logits_ = logits_model(x)
evidence_ = model.exp_evidence(logits_)
tot_ev = tf.reduce_sum(evidence_,axis=1, keepdims=True)
prediction = tf.argmax(logits_, axis=1, output_type=y.dtype)
probs = (evidence_+1)/ (tot_ev+2)
match = tf.equal(prediction, y)
no_match = tf.not_equal(prediction, y)
ev_succ = tf.reduce_mean(tf.boolean_mask(tot_ev,match))
ev_fail = tf.reduce_mean(tf.boolean_mask(tot_ev, no_match))
exp_entropy_ = model.tf_dirichlet_expected_entropy(evidence_+1)
u = 2. / (tot_ev+2)
u_succ = tf.boolean_mask(u,match)
u_fail = tf.boolean_mask(u, no_match)
print('u succ:', ev_succ.numpy().mean(), 'ev_fail:', ev_fail.numpy().mean(),
'ent:', exp_entropy_.numpy().mean(), 'u:', u.numpy().mean(), 'mean_p:', probs.numpy()[:,0].mean())
# print('y: ', y.shape)
# print('prediction: ', prediction.shape)
test_accuracy.update_state(prediction, y)
# Log statistics
if epoch % 1 == 0:
template = "Epoch {:03d}: Train loss: {:.3f}, Train entropy: {:.3f}, " \
"Train accuracy: {:.3%}, Test accuracy: {:.3%}"
print(template.format(epoch,
train_loss_avg.result(),
train_entropy_avg.result(),
train_accuracy.result(),
test_accuracy.result()))
#
# Make a checkpoint
if epoch % 1 == 0:
root.save(checkpoint_prefix)
# root.restore(tf.train.latest_checkpoint(checkpoint_dir))
#
# end epoch
tf.summary.scalar('train_loss_avg', train_loss_avg.result())
tf.summary.scalar('train_entropy_avg', train_entropy_avg.result())
tf.summary.scalar('train_accuracy', train_accuracy.result())
tf.summary.scalar('test_accuracy', test_accuracy.result())
train_loss_results.append(train_loss_avg.result())
train_loss_results.append(train_entropy_avg.result())
train_accuracy_results.append(train_accuracy.result())
test_accuracy_results.append(test_accuracy.result())
global_step.assign_add(1)
#
# Make a final checkpoint
root.save(checkpoint_prefix)
# history = classifier.fit(
# ds_train,
# epochs=FLAGS.num_epochs,
# validation_data=ds_validation,
# class_weight=dtask.class_weight(),
# callbacks=[
# tfk.callbacks.LambdaCallback(on_epoch_end=lambda epoch, logs: increment_epoch()),
# tfk.callbacks.TensorBoard(
# log_dir=os.path.join(out_dir, 'fit'),
# update_freq="epoch",
# write_graph=True,
# histogram_freq=1,
# ),
# tfk.callbacks.ModelCheckpoint(
# filepath=os.path.join(out_dir, "checkpoints", "weights-{epoch}.ckpt"),
# verbose=1,
# save_weights_only=True,
# )
# ],
# )
# plotting.tfk_history(history, output_dir=os.path.join(out_dir, "history"))
#
# Plot history
fig, axes = plt.subplots(2, sharex=True, figsize=(12, 8))
fig.suptitle('Training Metrics')
#
axes[0].set_ylabel("Loss", fontsize=14)
axes[0].plot(train_loss_results)
#
axes[1].set_ylabel("Accuracy", fontsize=14)
axes[1].set_xlabel("Epoch", fontsize=14)
axes[1].plot(train_accuracy_results, label='train')
axes[1].plot(test_accuracy_results, label='test')
axes[1].legend(loc='lower right')
plt.savefig('tmp.png')
def main(argv):
# print(argv)
# print(FLAGS)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
out_dir = os.path.join(FLAGS.output_dir, 'EDL', current_time)
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3),
realworld=(512, 512, 3))[FLAGS.level]
hparams = dict(dropout_rate=FLAGS.dropout_rate,
num_base_filters=FLAGS.num_base_filters,
learning_rate=FLAGS.learning_rate,
l2_reg=FLAGS.l2_reg,
input_shape=input_shape)
classifier = VGGDrop(**hparams)
# classifier.summary()
print('********** Output dir: {} ************'.format(out_dir))
if FLAGS.model_dir:
FLAGS.num_epochs = 0
latest = tf.train.latest_checkpoint(FLAGS.model_dir)
print('********** Loading checkpoint weights: {}'.format(latest))
classifier.load_weights(latest)
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
ds_train, ds_validation, ds_test = dtask.datasets
#################
# Training Loop #
#################
history = classifier.fit(
ds_train,
epochs=FLAGS.num_epochs,
validation_data=ds_validation,
class_weight=dtask.class_weight(),
callbacks=[
tfk.callbacks.TensorBoard(
log_dir=os.path.join(out_dir, "tensorboard"),
update_freq="epoch",
write_graph=True,
histogram_freq=1,
),
tfk.callbacks.ModelCheckpoint(
filepath=os.path.join(out_dir, "checkpoints",
"weights-{epoch}.ckpt"),
verbose=1,
save_weights_only=True,
)
],
)
plotting.tfk_history(history, output_dir=os.path.join(out_dir, "history"))
##############
# Evaluation #
##############
additional_metrics = []
try:
import sail.metrics
additional_metrics.append(
('ECE', sail.metrics.GPleissCalibrationError()))
except ImportError:
import warnings
warnings.warn('Could not import SAIL metrics.')
dtask.evaluate(functools.partial(predict,
model=classifier,
type=FLAGS.uncertainty),
dataset=ds_test,
output_dir=os.path.join(out_dir, 'evaluation'),
additional_metrics=additional_metrics)
def saved_model(image_path,
name,
model_id,
scale='linear',
labels=None,
testing_dataset=None,
batching=False,
validation=None):
image = cv2.imread(image_path)
ZipFile(load_gdrive_file(model_id,
'zip')).extractall('/tmp/extracted_module')
tf.compat.v1.enable_resource_variables()
net = tf.saved_model.load('/tmp/extracted_module')
def eval_func(image):
if batching:
image = tf.expand_dims(image, 0)
out = net.signatures['serving_default'](tf.cast(image, tf.float32))
for key, val in out.items():
print(key, val.shape, flush=True)
return out['anomaly_score']
out = eval_func(image).numpy()
if batching:
out = out[0]
if scale == 'exp':
out = np.exp(out)
elif scale == 'log':
out = np.log(out)
min_val, max_val = out.min(), out.max()
disp = (out - min_val) / (max_val - min_val)
disp = 255 - (np.clip(disp, 0, 1) * 255).astype('uint8')
directory, filename = os.path.split(image_path)
if labels is None:
cv2.imwrite(
os.path.join(directory, f'{filename.split(".")[0]}_{name}.jpg'),
disp)
return
# since we have labels, check for the accuracy
def data_generator():
rgb = cv2.imread(image_path).astype('float32')
label = cv2.imread(labels, cv2.IMREAD_ANYDEPTH).astype('int32')
yield {'image_left': rgb, 'mask': label}
data = tf.data.Dataset.from_generator(data_generator, {
'image_left': tf.float32,
'mask': tf.int32
})
fs = bdlb.load(benchmark='fishyscapes', download_and_prepare=False)
metrics = fs.evaluate(eval_func, data)
print(metrics['AP'], flush=True)
cv2.imwrite(
os.path.join(
directory,
f'{filename.split(".")[0]}_{name}_AP{100 * metrics["AP"]:.2f}.jpg'
), disp)
def main(argv):
print(argv)
print(FLAGS)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
out_dir = os.path.join(FLAGS.output_dir, 'MCdropout', current_time)
file_writer = tf.summary.create_file_writer(
os.path.join(os.path.join(out_dir, 'summary')))
file_writer.set_as_default()
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3),
realworld=(512, 512, 3))[FLAGS.level]
hparams = dict(dropout_rate=FLAGS.dropout_rate,
num_base_filters=FLAGS.num_base_filters,
learning_rate=FLAGS.learning_rate,
l2_reg=FLAGS.l2_reg,
input_shape=input_shape)
classifier = VGGDrop(**hparams)
classifier.summary()
print('********** Output dir: {} ************'.format(out_dir))
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
ds_train, ds_validation, ds_test = dtask.datasets
#################
# Training Loop #
#################
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
history = classifier.fit(
ds_train,
epochs=FLAGS.num_epochs,
validation_data=ds_validation,
class_weight=dtask.class_weight(),
callbacks=[
tfk.callbacks.TensorBoard(
log_dir=os.path.join(out_dir, 'fit'),
update_freq="epoch",
write_graph=True,
histogram_freq=1,
),
tfk.callbacks.ModelCheckpoint(
filepath=os.path.join(out_dir, "checkpoints",
"weights-{epoch}.ckpt"),
verbose=1,
save_weights_only=True,
)
],
)
plotting.tfk_history(history, output_dir=os.path.join(out_dir, "history"))
##############
# Evaluation #
##############
dtask.evaluate(functools.partial(predict,
model=classifier,
num_samples=FLAGS.num_mc_samples,
type=FLAGS.uncertainty),
dataset=ds_test,
output_dir=os.path.join(out_dir, 'evaluation'))
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])
# dissimilarity pre-process
self.vgg_diff = VGG19_difference().cuda()
self.base_transforms_diss = transforms.Compose([
transforms.Resize(size=(256, 512), interpolation=Image.NEAREST),
transforms.ToTensor()
])
self.norm_transform_diss = transforms.Compose([
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
]) # imageNet normamlization
self.to_pil = ToPILImage()
if __name__ == '__main__':
import bdlb
# define fishyscapes test parameters
fs = bdlb.load(benchmark="fishyscapes")
# automatically downloads the dataset
data = fs.get_dataset('Static')
detector = AnomalyDetector(True)
metrics = fs.evaluate(detector.estimator_worker, data)
print('My method achieved {:.2f}% AP'.format(100 * metrics['AP']))
print('My method achieved {:.2f}% FPR@95TPR'.format(100 *
metrics['FPR@95%TPR']))
print('My method achieved {:.2f}% auroc'.format(100 * metrics['auroc']))
def main(argv):
# print(argv)
# print(FLAGS)
current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
out_dir = os.path.join(FLAGS.output_dir, 'TS', current_time)
##########################
# Hyperparmeters & Model #
##########################
input_shape = dict(medium=(256, 256, 3),
realworld=(512, 512, 3))[FLAGS.level]
hparams = dict(dropout_rate=FLAGS.dropout_rate,
num_base_filters=FLAGS.num_base_filters,
learning_rate=FLAGS.learning_rate,
l2_reg=FLAGS.l2_reg,
input_shape=input_shape)
classifier = model.VGG(**hparams)
# classifier.summary()
print('********** Output dir: {} ************'.format(out_dir))
latest = tf.train.latest_checkpoint(FLAGS.model_dir)
print('********** Loading checkpoint weights: {}'.format(latest))
classifier.load_weights(latest)
#############
# Load Task #
#############
dtask = bdlb.load(
benchmark="diabetic_retinopathy_diagnosis",
level=FLAGS.level,
batch_size=FLAGS.batch_size,
download_and_prepare=False, # do not download data from this script
)
ds_train, ds_validation, ds_test = dtask.datasets
###############
# Build model #
###############
binary_prob_to_multi = model.BinaryProbToMulticlass()
inv_softmax = model.InverseSoftmaxFixedMean(mean=1.)
ts_layer = model.TemperatureScaling()
multi_to_binary_prob = model.MulticlassToBinaryProb()
ts_model = tfkm.Sequential([
classifier, binary_prob_to_multi, inv_softmax, ts_layer,
tfkl.Softmax(), multi_to_binary_prob
])
ts_model.build(input_shape=hparams['input_shape'])
classifier.summary()
ts_model.summary()
print('classifier: ', classifier.get_layer(1).output_shape)
print('ts_model: ', ts_model.get_layer(1).output_shape)
########################
# Optimise temperature #
########################
#
# Calculate logits for validation set
y_true = []
logits = []
for x, y in ds_validation:
p = classifier(x)
p_multi = binary_prob_to_multi(p)
logit = inv_softmax(p_multi)
logits.append(logit)
y_true.append(tf.one_hot(y, depth=2))
y_true = tf.concat(y_true, axis=0)
logits = tf.concat(logits, axis=0)
#
# Optimise temperature
ts_layer.optimise_temperature(y_true, logits)
##############
# Evaluation #
##############
additional_metrics = []
try:
import sail.metrics
additional_metrics.append(
('ECE', sail.metrics.GPleissCalibrationError()))
except ImportError:
import warnings
warnings.warn('Could not import SAIL metrics.')
dtask.evaluate(functools.partial(model.predict,
model=ts_model,
type=FLAGS.uncertainty),
dataset=ds_test,
output_dir=os.path.join(out_dir, 'evaluation'),
additional_metrics=additional_metrics)
print('Temperature: ', ts_layer.temperature.value())
