def get_dataset(model, dataset_name):
logger.debug("Loading dataset")
flat = len(model._layers[0].batch_input_shape) == 2
if dataset_name.upper() == "EMNIST":
return Datasets.EMNIST(flatten=flat)
else:
return Datasets.MNIST(flatten=flat)
def exp_low_amax():
model = keras.models.load_model('mnist_models/mnist_cnn0.h5')
dataset = Datasets.EMNIST(flatten=False)
X = np.concatenate((dataset.get_range(
'test', 'x', 0), dataset.get_range('train', 'x', 0)),
axis=0)
Y = np.concatenate((dataset.get_range(
'test', 'y', 0), dataset.get_range('train', 'y', 0)),
axis=0)
results = find_low_amax(model, X, Y)
def exp_confusion():
print("exp_confusion")
for model_f in sorted(glob.glob("mnist_models/*.h5")):
print('-' * 80)
print(model_f)
model = keras.models.load_model(model_f)
for dataset in [Datasets.MNIST(), Datasets.EMNIST()]:
X = np.concatenate((dataset.get_range('test', 'x', 0),\
dataset.get_range('train', 'x', 0)), axis=0)
Y = np.concatenate((dataset.get_range('test', 'y', 0),\
dataset.get_range('train', 'y', 0)), axis=0)
print(model.evaluate(X, Y))
Y_pred = map(np.argmax, model.predict(X))
cm = confusion_matrix(map(np.argmax, Y), Y_pred)
print(cm)
def exp_coverage(dataset_name, model_path):
logger.info("Loading model {}".format(model_path))
model = keras.models.load_model(model_path)
flat = len(model._layers[0].batch_input_shape) == 2
logger.info("Loading dataset")
dataset = None
if dataset_name.upper() == "EMNIST":
dataset = Datasets.EMNIST(flatten=flat)
else:
dataset = Datasets.MNIST(flatten=flat)
f_outs = "csvs/neuron_cov.csv"
logger.info("Initializing KerasCoverage")
kcov = KerasCoverage(model, dataset)
logger.info("Measuring coverage")
kcov.run([dataset.get_range('test', 'x', 0, 10000)])
kcov.measure_coverage('neuron')
kcov.save_outs(filename=f_outs)
print(len(kcov.coverage['neuron'].columns))
logger.info("Calculating entropy from {}".format(f_outs))
entropy = KerasEntropy(kcov.coverage['neuron'])
entropy.measure()