cpu_use = args.cpu
# Save the command run
if not os.path.isdir('CMDs'):
os.mkdir('CMDs')
with open('CMDs/precision_recall_linear_classifier.cmd', 'a') as f:
f.write(' '.join(sys.argv) + '\n')
# Get device
if cpu_use == 'yes':
device = torch.device('cpu')
else:
device = get_default_device()
# Load the Sentiment Classifier model
model = BERTGrader()
model.load_state_dict(
torch.load(model_path, map_location=torch.device('cpu')))
model.eval()
# Load the Adv Attack Detector model
detector = LayerClassifier(num_comps)
detector.load_state_dict(
torch.load(detector_path, map_location=torch.device('cpu')))
detector.eval()
# Load the eigenvectors for PCA decomposition and the correction mean
eigenvectors = torch.load(eigenvectors_path)
correction_mean = torch.load(correction_mean_path)
# Prepare test input tensors (mapped to pca components)
train_grades_file = args.TRAIN_GRADES
test_grades_file = args.TEST_GRADES
attack_phrase = args.ATTACK
out_file = args.OUT
head_num = args.head_num
num_comps = args.num_comps
start = args.start
# Save the command run
if not os.path.isdir('CMDs'):
os.mkdir('CMDs')
with open('CMDs/pca_component_comparison_plot_comps.cmd', 'a') as f:
f.write(' '.join(sys.argv)+'\n')
# Load the model
model = BERTGrader()
model.load_state_dict(torch.load(model_path))
model.eval()
# Use training data to get eigenvector basis of CLS token at correct layer
embeddings, _ = get_head_embedding(train_data_file, train_grades_file, model, attack_phrase='', head_num=head_num)
with torch.no_grad():
correction_mean = torch.mean(embeddings, dim=0)
cov = get_covariance_matrix(embeddings)
e, v = get_e_v(cov)
# Map test data to PCA components
embeddings, labels = get_head_embedding(test_data_file, test_grades_file, model, attack_phrase=attack_phrase, head_num=head_num)
pca_comps = get_pca_principal_components(v, correction_mean, embeddings, num_comps, start)
# Plot all the data
torch.manual_seed(seed)
# Save the command run
if not os.path.isdir('CMDs'):
os.mkdir('CMDs')
with open('CMDs/linear_pca_classifier.cmd', 'a') as f:
f.write(' '.join(sys.argv) + '\n')
# Get device
if cpu_use == 'yes':
device = torch.device('cpu')
else:
device = get_default_device()
# Load the model
model = BERTGrader()
model.load_state_dict(
torch.load(model_path, map_location=torch.device('cpu')))
model.eval()
# Use training data to get eigenvector basis of head embedding
embeddings, _ = get_head_embedding(train_data_file,
train_grades_file,
model,
attack_phrase='',
head_num=head_num)
with torch.no_grad():
correction_mean = torch.mean(embeddings, dim=0)
cov = get_covariance_matrix(embeddings)
e, v = get_e_v(cov)
mask_val = mask[:val_size]
labels_val = labels[:val_size]
input_ids_train = input_ids[val_size:]
mask_train = mask[val_size:]
labels_train = labels[val_size:]
# Use dataloader to handle batches
train_ds = TensorDataset(input_ids_train, mask_train, labels_train)
val_ds = TensorDataset(input_ids_val, mask_val, labels_val)
train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True)
val_dl = DataLoader(val_ds, batch_size=batch_size)
# initialise grader
model = BERTGrader()
model.to(device)
# Optimizer
optimizer = torch.optim.AdamW(model.parameters(), lr=lr, eps=1e-8)
# Scheduler
scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[sch])
# Criterion
criterion = torch.nn.MSELoss(reduction='mean')
# Train
for epoch in range(epochs):
# Get the device
device = get_default_device()
# Load the data
input_ids, mask, labels, speakerids = get_data(responses_file,
grades_file,
part=part)
print(len(labels))
test_ds = TensorDataset(input_ids, mask, labels)
test_dl = DataLoader(test_ds, batch_size=batch_size)
# Load the models
models = []
for model_path in model_paths:
model = BERTGrader()
model.load_state_dict(torch.load(model_path))
model.to(device)
models.append(model)
targets = None
all_preds = []
for model in models:
preds, targets = eval(test_dl, model, device)
all_preds.append(preds)
# Get single stats
mse_mean, mse_std, pcc_mean, pcc_std, avg_mean, avg_std, less05_mean, less05_std, less1_mean, less1_std = get_single_stats(
all_preds, targets)
print("STATS FOR ", model_paths)