def eval_fgsm_bnn(model,
data,
inv_factors,
estimator='kfac',
samples=30,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_state = copy.deepcopy(model.state_dict())
mean_predictions = 0
samples = tqdm.tqdm(range(samples), disable=not verbose)
for sample in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
sample_and_replace_weights(model, inv_factors, estimator)
predictions, labels, _ = eval_fgsm(model, data, epsilon, stats=False, device=device, verbose=False)
mean_predictions += predictions
model.load_state_dict(mean_state)
mean_predictions /= len(samples)
if stats:
acc = accuracy(mean_predictions, labels)
ece1 = 100 * expected_calibration_error(mean_predictions, labels)[0]
ece2 = 100 * calibration_curve(mean_predictions, labels)[0]
nll = negative_log_likelihood(mean_predictions, labels)
ent = predictive_entropy(mean_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return mean_predictions, labels, stats_dict
def eval_bnn(model,
dataset,
inv_factors,
estimator='kfac',
samples=30, stats=False,
device=torch.device('cuda'),
verbose=True):
model.eval()
mean_state = copy.deepcopy(model.state_dict())
mean_predictions = 0
stats_list = {"acc": [], "ece": [], "nll": [], "ent": []}
with torch.no_grad():
samples = tqdm.tqdm(range(samples), disable=not verbose)
for sample in samples:
samples.set_postfix({'RAM': ram(), 'VRAM': vram()})
sample_and_replace_weights(model, inv_factors, estimator)
predictions, labels = eval_nn(model, dataset, device)
mean_predictions += predictions
model.load_state_dict(mean_state)
if stats:
running_mean = mean_predictions / (sample + 1)
stats_list["acc"].append(accuracy(running_mean, labels))
stats_list["ece"].append(100 * expected_calibration_error(running_mean, labels)[0])
stats_list["nll"].append(negative_log_likelihood(predictions, labels))
stats_list["ent"].append(predictive_entropy(running_mean, mean=True))
mean_predictions /= len(samples)
if verbose:
print(f"Accuracy: {accuracy(mean_predictions, labels):.2f}% | ECE: {100 * expected_calibration_error(mean_predictions, labels)[0]:.2f}%")
return mean_predictions, labels, stats_list
def eval_fgsm(model,
data,
epsilon=0.1,
stats=True,
device=torch.device('cuda'),
verbose=True):
model.eval()
logits_list = torch.Tensor().to(device)
labels_list = torch.LongTensor()
stats_dict = None
data = tqdm.tqdm(data, disable=not verbose or len(data) == 1)
for images, labels in data:
data.set_postfix({'RAM': ram(), 'VRAM': vram()})
adv_images = datasets.fgsm(model, images.to(device, non_blocking=True), labels.to(device, non_blocking=True),
epsilon=epsilon)
with torch.no_grad():
adv_logits = model(adv_images)
logits_list = torch.cat([logits_list, adv_logits])
labels_list = torch.cat([labels_list, labels])
adv_predictions = torch.nn.functional.softmax(logits_list, dim=1).detach().cpu().numpy()
labels = labels_list.numpy()
if stats:
acc = accuracy(adv_predictions, labels)
ece1 = 100 * expected_calibration_error(adv_predictions, labels)[0]
ece2 = 100 * calibration_curve(adv_predictions, labels)[0]
nll = negative_log_likelihood(adv_predictions, labels)
ent = predictive_entropy(adv_predictions, mean=True)
stats_dict = {"eps": epsilon, "acc": acc, "ece1": ece1, "ece2": ece2, "nll": nll, "ent": ent}
if verbose:
print(f"Step: {epsilon:.2f} | Adv. Entropy: {stats_dict['ent']:.2f} | Adv. Accuracy: {stats_dict['acc']:.2f}%")
return adv_predictions, labels, stats_dict
# For songs sampling
"TEMPERATURE": 1,
"TAKE_MAX_PROBABLE": False,
"LIMIT_LEN": 300
}
print(config)
# model = VanillaRNN(config["VOCAB_SIZE"], config["HIDDEN"], config["VOCAB_SIZE"]).to(get_device())
model = LSTMSimple(config["VOCAB_SIZE"], config["HIDDEN"],
config["VOCAB_SIZE"]).to(get_device())
criterion = CrossEntropyLoss()
# Fit Model
fit(model, train_encoded, val_encoded, config)
# Report NLL for validation and test
nll_val = negative_log_likelihood(model, val_encoded, criterion, config)
nll_test = negative_log_likelihood(model, test_encoded, criterion, config)
print("NLL Validation: {}".format(nll_val))
print("NLL Test: {}".format(nll_test))
# Save error plot to file
save_loss_graph(model)
# Save model to file
print("Saving model...")
now = datetime.now().strftime('%Y-%m-%d-%H-%M')
torch.save(model.state_dict(), "model" + now + ".pth")
print("Saved!")
def objective(**params):
norms = list()
scales = list()
for f in f_norms:
if args.layer:
# Closest to max
if abs(f_norms.max() - f) < abs(f_norms.min() - f) and abs(
f_norms.max() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
# Closest to min
elif abs(f_norms.min() - f) < abs(f_norms.max() - f) and abs(
f_norms.min() - f) < abs(f_norms.mean() - f):
norms.append(10**params['norm1'])
scales.append(10**params['scale1'])
# Closest to mean
else:
norms.append(10**params['norm2'])
scales.append(10**params['scale2'])
else:
norms.append(10**params['norm0'])
scales.append(10**params['scale0'])
if args.layer:
print(
tabulate.tabulate(
{
'Layer': np.arange(len(factors)),
'F-Norm:': f_norms,
'Norms': norms,
'Scales': scales
},
headers='keys',
numalign='right'))
else:
print("Norm:", norms[0], "Scale:", scales[0])
try:
inv_factors = invert_factors(factors, norms,
args.pre_scale * scales,
args.estimator)
except (RuntimeError, np.linalg.LinAlgError):
print(f"Error: Singular matrix")
return 200
predictions, labels, _ = eval_bnn(model,
val_loader,
inv_factors,
args.estimator,
args.samples,
stats=False,
device=args.device,
verbose=False)
err = 100 - accuracy(predictions, labels)
ece = 100 * expected_calibration_error(predictions, labels)[0]
nll = negative_log_likelihood(predictions, labels)
ent = predictive_entropy(predictions, mean=True)
stats["norms"].append(norms)
stats["scales"].append(scales)
stats["acc"].append(100 - err)
stats["ece"].append(ece)
stats["nll"].append(nll)
stats["ent"].append(ent)
stats["cost"].append(err + ece)
print(
f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}"
)
return err + ece
def build_model(self):
hidNum = self.conf.hidNum
visNum = self.dataset.sensor_num()
labNum = self.dataset.total_combined_acts()
self.x = tf.placeholder(tf.float32, [None, self.max_len, visNum])
self.y = tf.placeholder(tf.float32, [None, self.max_len, labNum])
self.lr = tf.placeholder(tf.float32, shape=[])
# This is useful for batch
mask = tf.sign(tf.reduce_max(tf.abs(self.y), reduction_indices=2))
lens = length(self.x)
# Choose for cell type
if self.conf.cell_type == "BasicRNN":
mycell = BasicRNNCell
elif self.conf.cell_type == "LSTM":
mycell = LSTMCell
elif self.conf.cell_type == "GRU":
mycell = GRUCell
else:
raise ValueError("cell type is not specified!!!")
s, _ = tf.nn.dynamic_rnn(mycell(hidNum),
self.x,
dtype=tf.float32,
sequence_length=lens)
s = tf.reshape(s, [-1, hidNum])
with tf.variable_scope("softmax_layer"):
weights = tf.get_variable(
"softmax_w", [hidNum, labNum],
initializer=tf.truncated_normal_initializer(stddev=1e-1))
biases = tf.get_variable("softmax_b", [labNum],
initializer=tf.constant_initializer(0.0))
o = tf.matmul(s, weights) + biases
o = tf.reshape(o, [-1, self.max_len, labNum])
pred = tf.argmax(o, 2)
# This is negative log likelihood loss
cost = negative_log_likelihood(o, self.y, lens, mask)
# This is sequence2sequence loss
"""
cost = tf.contrib.seq2seq.sequence_loss(
o,
tf.argmax(self.y,axis=2),
tf.ones([self.conf.batch_size, self.conf.num_steps], dtype=tf.float32), average_across_timesteps=False,average_across_batch=True)
cost = tf.reduce_sum(cost)
"""
# regularization --> dont use now
"""
l2 = self.conf.weight_decay*sum(tf.nn.l2_loss(tf_var)
for tf_var in tf.trainable_variables()
if not ("Bias" in tf_var.name or "softmax_b" in tf_var.name))
cost += l2
"""
return cost