def show(model, tokenizer, text):
vis_html = """
<div id='vis'></div>
"""
display(HTML(vis_html))
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
vis_js = open(os.path.join(__location__, 'model_view.js')).read()
attn_data = get_attention_gpt2(model, tokenizer, text)
params = {'attention': attn_data, 'default_filter': "all"}
display(Javascript('window.params = %s' % json.dumps(params)))
display(Javascript(vis_js))
def show(model, tokenizer, text):
vis_html = """
<div id='vis'></div>
"""
display(HTML(vis_html))
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
vis_js = open(os.path.join(__location__, 'model_view_gpt2.js')).read()
attn_data = get_attention_gpt2(model, tokenizer, text)
att_json = json.dumps(attn_data)
display(Javascript('window.attention = %s' % att_json))
display(Javascript(vis_js))
def show(model, tokenizer, text):
vis_html = """
<span style="user-select:none">
Layer: <select id="layer"></select>
</span>
<div id='vis'></div>
"""
display(HTML(vis_html))
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
vis_js = open(os.path.join(__location__, 'head_view_gpt2.js')).read()
attn_data = get_attention_gpt2(model, tokenizer, text)
att_json = json.dumps(attn_data)
display(Javascript('window.attention = %s' % att_json))
display(Javascript(vis_js))
def test_gpt2_attn(self):
model = GPT2Model.from_pretrained('gpt2')
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
text = 'Bert is a yellow muppet character'
attn_data = get_attention_gpt2(model,
tokenizer,
text,
include_queries_and_keys=False)
tokens = ['B', 'ert', 'is', 'a', 'yellow', 'm', 'uppet', 'character']
self.assertEqual(attn_data['left_text'], tokens)
self.assertEqual(attn_data['right_text'], tokens)
seq_len = len(tokens)
layer = 0
head = 0
att_matrix = attn_data['attn'][layer][head]
for i in range(seq_len):
for j in range(seq_len):
if i >= j:
self.assertNotEqual(att_matrix[i][j], 0)
else:
self.assertEqual(att_matrix[i][j], 0)
sum_probs = sum(att_matrix[i])
self.assertAlmostEqual(sum_probs, 1, 4)
def show(model, tokenizer, text):
vis_html = """
<span style="user-select:none">
Layer: <select id="layer"></select>
Head: <select id="att_head"></select>
</span>
<div id='vis'></div>
"""
display(HTML(vis_html))
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
vis_js = open(os.path.join(__location__, 'neuron_view.js')).read()
attn_data = get_attention_gpt2(model,
tokenizer,
text,
include_queries_and_keys=True)
params = {
'attention': attn_data,
'default_filter': "all",
'bidirectional': False
}
display(Javascript('window.params = %s' % json.dumps(params)))
display(Javascript(vis_js))