def create_and_check_bert_model(
self,
config,
input_ids,
token_type_ids,
input_mask,
sequence_labels,
token_labels,
choice_labels,
):
model = BertModel(config=config)
model.eval()
sequence_output, pooled_output = model(input_ids, token_type_ids,
input_mask)
sequence_output, pooled_output = model(input_ids, token_type_ids)
sequence_output, pooled_output = model(input_ids)
result = {
"sequence_output": sequence_output,
"pooled_output": pooled_output
}
self.parent.assertListEqual(
list(result["sequence_output"].size()),
[self.batch_size, self.seq_length, self.hidden_size],
)
self.parent.assertListEqual(list(result["pooled_output"].size()),
[self.batch_size, self.hidden_size])
import optuna
# notif
webhook = f"https://hooks.slack.com/services/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"
# fix seed
np.random.seed(0)
sns.set()
model_type = 'bert-base-uncased'
tokenizer = BertTokenizer.from_pretrained(model_type)
config = BertConfig.from_pretrained(model_type)
config.output_hidden_states = True
config.output_attentions = True
model = BertModel(config)
model.eval()
def calc_hid_rep():
output_data_path = Path(f"probing_data/BERT/ST_neuron.npy")
if output_data_path.exists():
return
data_path = Path(f"probing_data/ST/ST-dev.json")
# Activate the neuron
with data_path.open("r", encoding="utf-8") as f:
data = json.load(f)
for i, data_i in data["data"].items(): # data_i = data["data"]["1"]
original_sentence = " ".join(data_i["word"])
