def show(model, tokenizer, text): vis_html = """ <div id='vis'></div> """ display(HTML(vis_html)) __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) vis_js = open(os.path.join(__location__, 'model_view.js')).read() attn_data = get_attention_gpt2(model, tokenizer, text) params = {'attention': attn_data, 'default_filter': "all"} display(Javascript('window.params = %s' % json.dumps(params))) display(Javascript(vis_js))
def show(model, tokenizer, text): vis_html = """ <div id='vis'></div> """ display(HTML(vis_html)) __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) vis_js = open(os.path.join(__location__, 'model_view_gpt2.js')).read() attn_data = get_attention_gpt2(model, tokenizer, text) att_json = json.dumps(attn_data) display(Javascript('window.attention = %s' % att_json)) display(Javascript(vis_js))
def show(model, tokenizer, text): vis_html = """ <span style="user-select:none"> Layer: <select id="layer"></select> </span> <div id='vis'></div> """ display(HTML(vis_html)) __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) vis_js = open(os.path.join(__location__, 'head_view_gpt2.js')).read() attn_data = get_attention_gpt2(model, tokenizer, text) att_json = json.dumps(attn_data) display(Javascript('window.attention = %s' % att_json)) display(Javascript(vis_js))
def test_gpt2_attn(self): model = GPT2Model.from_pretrained('gpt2') tokenizer = GPT2Tokenizer.from_pretrained('gpt2') text = 'Bert is a yellow muppet character' attn_data = get_attention_gpt2(model, tokenizer, text, include_queries_and_keys=False) tokens = ['B', 'ert', 'is', 'a', 'yellow', 'm', 'uppet', 'character'] self.assertEqual(attn_data['left_text'], tokens) self.assertEqual(attn_data['right_text'], tokens) seq_len = len(tokens) layer = 0 head = 0 att_matrix = attn_data['attn'][layer][head] for i in range(seq_len): for j in range(seq_len): if i >= j: self.assertNotEqual(att_matrix[i][j], 0) else: self.assertEqual(att_matrix[i][j], 0) sum_probs = sum(att_matrix[i]) self.assertAlmostEqual(sum_probs, 1, 4)
def show(model, tokenizer, text): vis_html = """ <span style="user-select:none"> Layer: <select id="layer"></select> Head: <select id="att_head"></select> </span> <div id='vis'></div> """ display(HTML(vis_html)) __location__ = os.path.realpath( os.path.join(os.getcwd(), os.path.dirname(__file__))) vis_js = open(os.path.join(__location__, 'neuron_view.js')).read() attn_data = get_attention_gpt2(model, tokenizer, text, include_queries_and_keys=True) params = { 'attention': attn_data, 'default_filter': "all", 'bidirectional': False } display(Javascript('window.params = %s' % json.dumps(params))) display(Javascript(vis_js))