def predict(self, dataSubType):
"""Predict the answers for a dataSubType. The method loads the data
first.
Parameters
----------
dataSubType : str
"val2014" for example.
"""
# File names for the annotations and questions
self.annFile_test = ann_file(self.dataDir, self.versionType,
self.dataType, dataSubType)
self.quesFile_test = ques_file(self.dataDir, self.versionType,
self.taskType, self.dataType, dataSubType)
# Initialize VQA api for the dataSubType on which to make predictions
# on the answers
self.vqa_test = VQA(self.annFile_test, self.quesFile_test)
question_ids_test = [annotation["question_id"] for annotation
in self.vqa_test.dataset["annotations"]]
# Result list [{"answer": "yes", "question_id": 1}]
res = [{"answer": "yes", "question_id": question_id}
for question_id in question_ids_test]
# Save the results
self.results_file = save_results(res, self.dataDir, self.taskType,
self.dataType, dataSubType,
self.__class__.__name__)
def predict(self, dataSubType):
"""Predict the answers for a dataSubType.
Parameters
----------
dataSubType : str
"val2014" for example.
"""
train_qID_to_imgID = {
dic["question_id"]: dic["image_id"]
for dic in self.questions_train
}
# Dicionary mapping q_id in train to its answer
answers_train = {
annotation["question_id"]: Counter(
(annotation["answers"][i]["answer"]
for i in range(10))).most_common(1)[0][0]
for annotation in self.vqa_train2014.dataset["annotations"]
}
# Result list [{"answer": "picked from NN image", "question_id": 1}]
res = [{
"answer":
answers_train[self.test_qid_to_nn_train_qids[q_id][
self.qID_to_img_nn_index[q_id]]],
"question_id":
q_id
} for q_id in self.qID_to_img_nn_index.keys()]
# Save the results
self.results_file = save_results(res, self.dataDir, self.taskType,
self.dataType, dataSubType,
self.__class__.__name__)
def predictions_to_dic(self, predictions, question_ids):
"""Turn the predictions from the model to the a result list as
required by the evaluation tool.
Parameters
----------
predictions : array, shape = [n, 1000]
Array resulting from the predict function, n is the number of
question/image pair for which we have predicted an answer.
question_ids : list(int)
Question ids corresponding to the questions on which we make
predictions.
Returns
-------
TYPE
Description
"""
# Predicted answers (as indices)
answers_ind_pred = np.argmax(predictions, axis=1)
# Predicted answers (as strings)
idx_to_answer = np.vectorize(self.idx_to_answer)
answers_pred = idx_to_answer(answers_ind_pred)
# Result list [{"answer": "yes", "question_id": 1}]
res = [{
"answer": answer,
"question_id": question_id
} for (answer, question_id) in zip(answers_pred, question_ids)]
# Save the results
self.results_file = save_results(res, self.dataDir, self.taskType,
self.dataType, self.dataSubTypeTest,
self.__class__.__name__)
def cv(data,
targets,
groups,
modfun,
rnn=False,
trans_tuple=None,
epochs=250,
folds=5,
batch_size=256,
val_batch_size=0,
stop_after=0,
name='',
counter=0,
plot=False,
balanced=False,
cropsize=0):
"""
Crossvalidation routinge for an RNN using extracted features on a basemodel
:param rnns: list with the following:
[rnnfun, [layernames], seqlen, batch_size]
:param stop_after: stop after x epochs without f1-improvement. 0 for no stopping
:param plot: True for plotting intermediate results and loss
:param counter: prefix for saving files. can be any number.
:param balanced: True if the generator should supply class-balanced batches
:param cropsize: Size that is randomly cropped for training (data augmentation)
:param ...: all others should be self-explanatory
:returns results: dictionary with all RNN results
"""
if val_batch_size == 0: val_batch_size = batch_size
input_shape = list((np.array(data[0])).shape) #train_data.shape
if cropsize != 0: input_shape[0] = cropsize
n_classes = targets.shape[1]
if type(modfun) == str:
wpath = modfun
modfun = False
gcv = GroupKFold(folds)
dict_id = modfun.__name__ + name if modfun else 'cnn' + '_' + name
results = {dict_id: []}
if rnn:
for lname in rnn['layers']:
results[name + '_' + lname] = []
for i, idxs in enumerate(gcv.split(groups, groups, groups)):
K.clear_session()
print('-----------------------------')
print('Starting fold {}: {}-{} at {}'.format(
i + 1, modfun.__name__ if modfun else 'cnn', name, time.ctime()))
train_idx, test_idx = idxs
sub_cv = GroupKFold(folds)
train_sub_idx, val_idx = sub_cv.split(groups[train_idx],
groups[train_idx],
groups[train_idx]).__next__()
val_idx = train_idx[val_idx]
train_idx = train_idx[train_sub_idx]
train_data = [data[i] for i in train_idx]
train_target = targets[train_idx]
train_groups = groups[train_idx]
val_data = [data[i] for i in val_idx]
val_target = targets[val_idx]
val_groups = groups[val_idx]
test_data = [data[i] for i in test_idx]
test_target = targets[test_idx]
test_groups = groups[test_idx]
if modfun:
model = modfun(input_shape, n_classes)
else:
fold = os.listdir(wpath)[i]
model = keras.models.load_model(os.path.join(wpath, fold))
modelname = model.name
lname = modelname
g_train = generator(train_data,
train_target,
batch_size * 2,
val=True,
cropsize=cropsize)
g_val = generator(val_data,
val_target,
batch_size * 2,
val=True,
cropsize=cropsize)
g_test = generator(test_data,
test_target,
batch_size * 2,
val=True,
cropsize=cropsize)
if balanced:
g = generator_balanced(train_data,
train_target,
batch_size,
cropsize=cropsize)
cb = Checkpoint_balanced(g_val,
g,
g_train,
verbose=1,
counter=counter,
groups=val_groups,
epochs_to_stop=stop_after,
plot=plot,
name='{}, {}, fold: {}'.format(
name, lname, i))
else:
g = generator(train_data,
train_target,
batch_size,
random=True,
cropsize=cropsize)
cb = Checkpoint_balanced(g_val,
verbose=1,
counter=counter,
groups=val_groups,
epochs_to_stop=stop_after,
plot=plot,
name='{}, {}, fold: {}'.format(
name, lname, i))
if modfun:
model.fit_generator(g,
g.n_batches,
epochs=epochs,
callbacks=[cb],
max_queue_size=1,
verbose=0)
y_pred = model.predict_generator(g_test,
g_test.n_batches,
max_queue_size=1)
y_true = g_test.Y
val_acc = cb.best_acc
val_f1 = cb.best_f1
test_acc = accuracy_score(np.argmax(y_true, 1), np.argmax(y_pred, 1))
test_f1 = f1_score(np.argmax(y_true, 1),
np.argmax(y_pred, 1),
average="macro")
confmat = confusion_matrix(np.argmax(y_true, 1), np.argmax(y_pred, 1))
if plot:
plt.subplot(2, 3, 5)
plt.cla()
tools.plot_results_per_patient(y_pred,
y_true,
test_groups,
fname='')
plt.title('Test Cases')
plt.subplot(2, 3, 6)
plt.cla()
tools.plot_confusion_matrix('',
confmat,
['W', 'S1', 'S2', 'SWS', 'REM'],
cbar=False)
plt.title('Test conf. Acc: {:.1f} F1: {:.1f}'.format(
test_acc * 100, test_f1 * 100))
plt.show()
plt.pause(0.0001)
results[dict_id].append(
[cb.best_acc, cb.best_f1, test_acc, test_f1, confmat])
if modfun: # only save if we calculated the results
try:
model.save(
os.path.join(
'.', 'weights',
str(counter) + name + model.name + '_' + str(i) +
"_{:.3f}-{:.3f}".format(test_acc, test_f1)))
except Exception as error:
print("Got an error while saving model: {}".format(error))
print(
'ANN results: val acc/f1: {:.5f}/{:.5f}, test acc/f1: {:.5f}/{:.5f}'
.format(cb.best_acc, cb.best_f1, test_acc, test_f1))
##########
if trans_tuple is not None:
trans_data, trans_target, trans_groups = trans_tuple
g_trans = generator(trans_data,
trans_target,
batch_size * 2,
val=True,
cropsize=cropsize)
y_trans = model.predict_generator(g_trans,
g_trans.n_batches,
max_queue_size=1)
t_trans = g_trans.Y
trans_acc = accuracy_score(np.argmax(t_trans, 1),
np.argmax(y_trans, 1))
trans_f1 = f1_score(np.argmax(t_trans, 1),
np.argmax(y_trans, 1),
average="macro")
print('Transfer ANN results: acc/f1: {:.5f}/{:.5f}'.format(
trans_acc, trans_f1))
##########
if rnn:
rnn_modelfun = rnn['model']
layernames = rnn['layers']
seq = rnn['seqlen']
rnn_bs = rnn['batch_size']
rnn_epochs = rnn['epochs']
stopafter_rnn = rnn['stop_after']
for lname in layernames:
extracted = get_activations(model,
train_data + val_data + test_data,
lname,
batch_size * 2,
cropsize=cropsize)
train_data_extracted = extracted[0:len(train_data)]
val_data_extracted = extracted[len(train_data
):len(train_data) +
len(val_data)]
test_data_extracted = extracted[len(train_data) +
len(val_data):]
assert (len(train_data) == len(train_data_extracted)) and (
len(test_data) == len(test_data_extracted)) and (
len(val_data) == len(val_data_extracted))
train_data_seq, train_target_seq, train_groups_seq = tools.to_sequences(
train_data_extracted,
train_target,
groups=train_groups,
seqlen=seq)
val_data_seq, val_target_seq, val_groups_seq = tools.to_sequences(
val_data_extracted,
val_target,
groups=val_groups,
seqlen=seq)
test_data_seq, test_target_seq, test_groups_seq = tools.to_sequences(
test_data_extracted,
test_target,
groups=test_groups,
seqlen=seq)
rnn_shape = list((np.array(train_data_seq[0])).shape)
neurons = 100
print('Starting RNN model with input from layer {}: {} at {}'.
format(lname, rnn_shape, time.ctime()))
rnn_model = rnn_modelfun(rnn_shape,
n_classes,
layers=2,
neurons=neurons,
dropout=0.3)
g_val = generator(val_data_seq,
val_target_seq,
rnn_bs * 2,
val=True)
g_test = generator(test_data_seq,
test_target_seq,
rnn_bs * 2,
val=True)
g_train = generator(train_data_seq,
train_target_seq,
batch_size * 2,
val=True)
if rnn['balanced']:
g = generator_balanced(train_data_seq, train_target_seq,
rnn_bs)
cb = Checkpoint_balanced(g_val,
g,
g_train,
verbose=1,
counter=counter,
groups=val_groups_seq,
epochs_to_stop=stopafter_rnn,
plot=plot,
name='{}, {}, fold: {}'.format(
name, lname, i))
else:
g = generator(train_data_seq, train_target_seq, rnn_bs)
cb = Checkpoint_balanced(g_val,
verbose=1,
counter=counter,
groups=val_groups_seq,
epochs_to_stop=stopafter_rnn,
plot=plot,
name='{}, {}, fold: {}'.format(
name, lname, i))
rnn_model.fit_generator(g,
g.n_batches,
epochs=rnn_epochs,
verbose=0,
callbacks=[cb],
max_queue_size=1)
y_pred = rnn_model.predict_generator(g_test,
g_test.n_batches,
max_queue_size=1)
y_true = g_test.Y
val_acc = cb.best_acc
val_f1 = cb.best_f1
test_acc = accuracy_score(np.argmax(y_true, 1),
np.argmax(y_pred, 1))
test_f1 = f1_score(np.argmax(y_true, 1),
np.argmax(y_pred, 1),
average="macro")
confmat = confusion_matrix(np.argmax(y_true, 1),
np.argmax(y_pred, 1))
try:
rnn_model.save(
os.path.join(
'.', 'weights',
str(counter) + name + lname + '_' + str(i) +
"_{:.3f}-{:.3f}".format(test_acc, test_f1)))
except Exception as error:
print("Got an error while saving model: {}".format(error))
if plot:
plt.subplot(2, 3, 5)
plt.cla()
tools.plot_results_per_patient(y_pred,
y_true,
test_groups_seq,
fname='')
plt.title('Test Cases')
plt.subplot(2, 3, 6)
plt.cla()
tools.plot_confusion_matrix(
'',
confmat, ['W', 'S1', 'S2', 'SWS', 'REM'],
cbar=False)
plt.title('Test conf. Acc: {:.1f} F1: {:.1f}'.format(
test_acc * 100, test_f1 * 100))
plt.show()
plt.pause(0.0001)
results[name + '_' + lname].append(
[val_acc, val_f1, test_acc, test_f1, confmat])
print(
'fold {}: val acc/f1: {:.5f}/{:.5f}, test acc/f1: {:.5f}/{:.5f}'
.format(i, cb.best_acc, cb.best_f1, test_acc, test_f1))
##########
if trans_tuple is not None:
trans_data, trans_target, trans_groups = trans_tuple
extracted = get_activations(model,
trans_data,
lname,
batch_size * 2,
cropsize=cropsize)
trans_data, trans_target, trans_groups = tools.to_sequences(
extracted,
trans_target,
groups=trans_groups,
seqlen=seq)
g_trans = generator(trans_data,
trans_target,
batch_size * 2,
val=True,
cropsize=0)
y_trans = rnn_model.predict_generator(g_trans,
g_trans.n_batches,
max_queue_size=1)
t_trans = g_trans.Y
trans_acc = accuracy_score(np.argmax(t_trans, 1),
np.argmax(y_trans, 1))
trans_f1 = f1_score(np.argmax(t_trans, 1),
np.argmax(y_trans, 1),
average="macro")
print(
'Transfer LSTM results: acc/f1: {:.5f}/{:.5f}'.format(
trans_acc, trans_f1))
##########
save_dict = {
'1 Number': counter,
'2 Time': time.ctime(),
'3 CV': '{}/{}.'.format(i + 1, folds),
'5 Model': lname,
'100 Comment': name,
'10 Epochs': epochs,
'11 Val acc': '{:.2f}'.format(val_acc * 100),
'12 Val f1': '{:.2f}'.format(val_f1 * 100),
'13 Test acc': '{:.2f}'.format(test_acc * 100),
'14 Test f1': '{:.2f}'.format(test_f1 * 100),
'Test Conf': str(confmat).replace('\n', '')
}
tools.save_results(save_dict=save_dict)
try:
with open('{}_{}_results.pkl'.format(counter, dict_id), 'wb') as f:
pickle.dump(results, f)
except Exception as e:
print("Error while saving results: ", e)
sys.stdout.flush()
return results
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 15 00:39:49 2021
@author: Selena
"""
import sys
sys.path.append('/home/new-ece/szc0173/liver/')
from unet import UNet
from tools import train_generator, test_generator, save_results, is_file, prepare_dataset, show_image
import tensorflow as tf
tf.test.gpu_device_name()
test_path = '/home/new-ece/szc0173/liver/test/'
save_path = '/home/new-ece/szc0173/liver/result2/'
model_weights_name='unet_weight_model3.hdf5'
# TODO: move to config .json files
img_height = 512
img_width = 512
img_size = (img_height, img_width)
model = UNet(
input_size = (img_width,img_height,1),
n_filters = 64,
pretrained_weights = model_weights_name
)
model.build()
test_gen = test_generator(test_path, 121, img_size)
results = model.predict_generator(test_gen,121,verbose=1)
save_results(save_path, results)
actor_loss, critic_a_loss, critic_b_loss = agent.learn()
# Log results
actor_loss_queue.append(actor_loss)
critic_a_loss_queue.append(critic_a_loss)
critic_b_loss_queue.append(critic_b_loss)
c_a_loss.append(np.mean(critic_a_loss_queue))
c_b_loss.append(np.mean(critic_b_loss_queue))
if reward != - 1:
# If the episode is over log results and print status messages
reward_mean.append(reward)
rewards.append(reward)
a_loss.append(np.mean(actor_loss_queue))
avg_rewards.append(np.mean(reward_mean))
episodeTime = time.time() - startTime
startTime = time.time()
print("\rEps: " + str(episode) + " rew: " + str(reward) + " std: " + str(std_reward) + " AvgRew:" + str(
np.mean(reward_mean)) + " crcloss: " + str(np.mean(critic_a_loss_queue)) + " " + str(np.mean(critic_b_loss_queue)) + " actor loss: " + str(np.mean(actor_loss_queue)) + " tim: " + str(episodeTime), end="")
if episode % 100 == 0:
print("")
episode += 1
if episode == 1000 or np.mean(reward_mean) > 40:
break
save_results(save_path, agent, settings, rewards, avg_rewards, a_loss, c_a_loss, c_b_loss)
def trojan_detector(args):
"""
Overview:
This detector uses a gradient-based trigger inversion approach with these steps
- calculate the trigger inversion loss and the gradient w.r.t. trigger embeddings
- get the top-k most promising candidates with the gradient from the previous step
- calculate the trigger inversion loss of each of the top-k candidates
- pick the best candidate, which gives us the lowest trigger inversion loss, as the new trigger
- repeat until convergence
At the end of this procedure we use the trigger inversion loss as the only predictive feature
in a logistic regression model. If our trigger inversion approach was successful, we get
a very low trigger inversion loss (close to zero)
Input:
In order to perform trigger inversion we need at least one clean model with the same architecture
and dataset as the evaluation model, as well as clean examples (i.e. without the trigger).
Output:
This function's output depends on wether this is meant to be inside of a submission container or not
If it's a submission, we output the probability that the evaluation model is trojaned
Otherwise, we output the trigger inversion loss, which we then use to train our classifier
"""
# print args
for arg in vars(args):
print(f'{arg}: {getattr(args, arg)}')
# load config
if args.is_manual_config:
config = {
'source_dataset': args.source_dataset,
'model_architecture': args.model_architecture
}
else:
config = tools.load_config(args.eval_model_filepath)
# load all models and get the clean_model_filepaths, which is used to get additional clean examples
models, clean_model_filepaths = tools.load_models(
config, args, CLEAN_TRAIN_MODELS_FILEPATH, CLEAN_TEST_MODELS_FILEPATH,
DEVICE)
# add hooks to pull the gradients out from all models when doing backward in the compute_loss function
def add_hooks_to_all_models(models):
def add_hooks_to_single_model(model, is_clean):
def find_word_embedding_module(classification_model):
word_embedding_tuple = [
(name, module)
for name, module in classification_model.named_modules()
if 'embeddings.word_embeddings' in name
]
assert len(word_embedding_tuple) == 1
return word_embedding_tuple[0][1]
module = find_word_embedding_module(model)
module.weight.requires_grad = True
if is_clean:
def extract_clean_grad_hook(module, grad_in, grad_out):
EXTRACTED_GRADS['clean_train'].append(grad_out[0])
module.register_backward_hook(extract_clean_grad_hook)
else:
def extract_grad_hook(module, grad_in, grad_out):
EXTRACTED_GRADS['eval'].append(grad_out[0])
module.register_backward_hook(extract_grad_hook)
add_hooks_to_single_model(models['eval'][0], is_clean=False)
for clean_model in models['clean_train']:
add_hooks_to_single_model(clean_model, is_clean=True)
add_hooks_to_all_models(models)
# get the word_embeddings for all the models
input_id_embeddings = tools.get_all_input_id_embeddings(models)
tokenizer = tools.load_tokenizer(args.is_submission,
args.tokenizer_filepath, config)
# get the candidate pool of tokens to start from at random initialization
total_cand_pool = tools.get_most_changed_embeddings(
input_id_embeddings, tokenizer, DEVICE)
def only_keep_avg_input_id_embeddings(input_id_embeddings):
return {
model_type: {
'avg': input_id_embeddings[model_type]['avg']
}
for model_type in list(input_id_embeddings.keys())
}
input_id_embeddings = only_keep_avg_input_id_embeddings(
input_id_embeddings)
# load and transform qa dataset
dataset = tools.load_qa_dataset(args.examples_dirpath,
args.scratch_dirpath,
clean_model_filepaths,
more_clean_data=args.more_clean_data)
print(f'dataset length: {len(dataset)}')
tokenized_dataset = tools.tokenize_for_qa(tokenizer, dataset)
tokenized_dataset = tools.select_examples_with_an_answer_in_context(
tokenized_dataset, tokenizer)
tokenized_dataset = tools.select_unique_inputs(tokenized_dataset)
df = pd.DataFrame()
# these are the behavior|insterions specific to QA
for behavior, insertion in [('self', 'both'), ('cls', 'both'),
('self', 'context'), ('cls', 'context'),
('cls', 'question')]:
best_test_loss = None
start_time = time.time()
args.trigger_behavior, args.trigger_insertion_type = behavior, insertion
# add a dummy trigger into input_ids, attention_mask, and token_type as well as provide masks for loss calculations
triggered_dataset = tools.get_triggered_dataset(
args, DEVICE, tokenizer, tokenized_dataset)
# insert trigger and populate baselines
def insert_trigger_and_populate_baselines():
tools.insert_new_trigger(
args, triggered_dataset,
torch.tensor([tokenizer.pad_token_id] * args.trigger_length,
device=DEVICE).long())
# zero out attention on trigger
tools.insert_new_trigger(args,
triggered_dataset,
torch.zeros(args.trigger_length,
device=DEVICE).long(),
where_to_insert='attention_mask')
# train loss to get train baseline
tools.compute_loss(args,
models,
triggered_dataset,
args.batch_size,
DEVICE,
LAMBDA,
with_gradient=False,
populate_baselines=True)
# test loss to get train baseline
models['clean_test'] = [
model.to(DEVICE, non_blocking=True)
for model in models['clean_test']
]
tools.compute_loss(args,
models,
triggered_dataset,
args.batch_size,
DEVICE,
LAMBDA,
with_gradient=False,
train_or_test='test',
populate_baselines=True)
models['clean_test'] = [
model.to(CPU, non_blocking=True)
for model in models['clean_test']
]
# add back attention
tools.insert_new_trigger(args,
triggered_dataset,
torch.ones(args.trigger_length,
device=DEVICE).long(),
where_to_insert='attention_mask')
insert_trigger_and_populate_baselines()
triggered_dataset = tools.take_best_k_inputs(triggered_dataset)
def put_embeds_on_device(device=DEVICE):
input_id_embeddings['eval']['avg'] = input_id_embeddings['eval'][
'avg'].to(device, non_blocking=True)
input_id_embeddings['clean_train']['avg'] = input_id_embeddings[
'clean_train']['avg'].to(device, non_blocking=True)
put_embeds_on_device()
for i in range(args.num_random_tries):
# get and insert new trigger
num_non_random_tries = args.num_random_tries // 2 if 'electra' in config[
'model_architecture'] else 2 * (args.num_random_tries // 3)
init_fn = args.trigger_init_fn if i < num_non_random_tries else 'random'
new_trigger = tools.initialize_trigger(args, init_fn,
total_cand_pool, tokenizer,
DEVICE)
tools.insert_new_trigger(args, triggered_dataset, new_trigger)
old_trigger, n_iter = torch.tensor([
randint(0, 20000) for _ in range(args.trigger_length)
]).to(DEVICE), 0
with autocast():
# main trigger inversion loop for a given random start
while not torch.equal(old_trigger,
new_trigger) and n_iter < args.max_iter:
n_iter += 1
old_trigger, old_loss = deepcopy(
new_trigger), tools.compute_loss(args,
models,
triggered_dataset,
args.batch_size,
DEVICE,
LAMBDA,
with_gradient=True)
@torch.no_grad()
def find_best_k_candidates_for_each_trigger_token(
num_candidates, tokenizer):
'''
equation 2: (embedding_matrix - trigger embedding)T @ trigger_grad
'''
embeds_shape = [
len(triggered_dataset['input_ids']), -1,
input_id_embeddings['eval']['avg'].shape[-1]
]
def get_mean_trigger_grads(eval_or_clean):
concat_grads = torch.cat(
EXTRACTED_GRADS[eval_or_clean])
grads_list = []
if args.trigger_insertion_type in [
'context', 'both'
]:
mean_context_grads_over_inputs = concat_grads[
triggered_dataset['c_trigger_mask']].view(
embeds_shape).mean(dim=0)
grads_list.append(
mean_context_grads_over_inputs)
if args.trigger_insertion_type in [
'question', 'both'
]:
mean_question_grads_over_inputs = concat_grads[
triggered_dataset['q_trigger_mask']].view(
embeds_shape).mean(dim=0)
grads_list.append(
mean_question_grads_over_inputs)
return torch.stack(grads_list).mean(dim=0)
eval_mean_trigger_grads = get_mean_trigger_grads(
'eval')
clean_train_mean_trigger_grads = get_mean_trigger_grads(
'clean_train')
eval_grad_dot_embed_matrix = torch.einsum(
"ij,kj->ik", (eval_mean_trigger_grads,
input_id_embeddings['eval']['avg']))
eval_grad_dot_embed_matrix[
eval_grad_dot_embed_matrix !=
eval_grad_dot_embed_matrix] = 1e2
clean_grad_dot_embed_matrix = torch.einsum(
"ij,kj->ik",
(clean_train_mean_trigger_grads,
input_id_embeddings['clean_train']['avg']))
clean_grad_dot_embed_matrix[
clean_grad_dot_embed_matrix !=
clean_grad_dot_embed_matrix] = 1e2
# fill nans
eval_grad_dot_embed_matrix[
eval_grad_dot_embed_matrix !=
eval_grad_dot_embed_matrix] = 1e2
clean_grad_dot_embed_matrix[
clean_grad_dot_embed_matrix !=
clean_grad_dot_embed_matrix] = 1e2
# weigh clean_train and eval dot products and get the smallest ones for each position
gradient_dot_embedding_matrix = eval_grad_dot_embed_matrix + LAMBDA * clean_grad_dot_embed_matrix
BANNED_TOKEN_IDS = [
tokenizer.pad_token_id, tokenizer.cls_token_id,
tokenizer.unk_token_id, tokenizer.sep_token_id,
tokenizer.mask_token_id
]
for token_id in BANNED_TOKEN_IDS:
gradient_dot_embedding_matrix[:, token_id] = 1e2
_, best_k_ids = torch.topk(
-gradient_dot_embedding_matrix,
num_candidates,
dim=1)
return best_k_ids
candidates = find_best_k_candidates_for_each_trigger_token(
args.num_candidates, tokenizer)
candidates = tools.add_candidate_variations_to_candidates(
old_trigger, args, candidates)
def clear_all_model_grads(models):
for model_type, model_list in models.items():
for model in model_list:
optimizer = optim.Adam(model.parameters())
optimizer.zero_grad(set_to_none=True)
for model_type in EXTRACTED_GRADS.keys():
EXTRACTED_GRADS[model_type] = []
clear_all_model_grads(models)
new_loss, new_trigger = tools.evaluate_and_pick_best_candidate(
args, models, DEVICE, LAMBDA, old_loss, old_trigger,
triggered_dataset, candidates, args.beam_size)
tools.insert_new_trigger(args, triggered_dataset,
new_trigger)
if args.is_submission == False:
tools.print_results_of_trigger_inversion_iterate(
n_iter, tokenizer, old_loss, new_loss, old_trigger,
new_trigger, LAMBDA, triggered_dataset)
del old_loss, candidates
new_test_loss = tools.get_new_test_loss(
args, models, triggered_dataset, DEVICE, LAMBDA)
if best_test_loss is None or best_test_loss[
'trigger_inversion_loss'] > new_test_loss[
'trigger_inversion_loss']:
best_trigger, best_train_loss, best_test_loss = deepcopy(
new_trigger), deepcopy(new_loss), deepcopy(new_test_loss)
if best_test_loss[
'trigger_inversion_loss'] < args.stopping_threshold:
break
torch.cuda.empty_cache()
df = tools.add_results_to_df(args, df, tokenizer, best_trigger,
best_train_loss, best_test_loss,
total_cand_pool, triggered_dataset,
start_time)
if args.is_submission:
tools.submit_results()
else:
tools.save_results(df, args)
epochs = 25
batch_size = 70
validation_split = 0.1
X, y, X_final = dataread()
X_train, X_test, y_train, y_test, prop_HF = datatreat_A1(
X,
y,
train_size=0.8,
Shuffle=True,
preprocess="None",
ratio="50/50",
balancing_method="SMOTEENN")
model = cnn_1()
history = model.fit(X_train,
y_train,
epochs=epochs,
batch_size=batch_size,
validation_split=validation_split)
save_model(model, id)
accuracy, roc, f1_macro, f1_wei = test_1(model, X_test, y_test, id)
save_results(id, 'datatreat_A1', 'cnn_1', epochs, batch_size, accuracy, roc,
f1_macro, f1_wei, validation_split)
plot_loss_acc_history(history, id, validation_split)