Python get_attention_gpt2の例

コード例 #1

ファイル: model_view_gpt2.py プロジェクト: pguntunur/SCOTUS-Embedding

def show(model, tokenizer, text):
    vis_html = """
      <div id='vis'></div>
    """
    display(HTML(vis_html))
    __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    vis_js = open(os.path.join(__location__, 'model_view.js')).read()
    attn_data = get_attention_gpt2(model, tokenizer, text)
    params = {'attention': attn_data, 'default_filter': "all"}
    display(Javascript('window.params = %s' % json.dumps(params)))
    display(Javascript(vis_js))

コード例 #2

ファイル: model_view_gpt2.py プロジェクト: huangpeng1126/bertviz

def show(model, tokenizer, text):
    vis_html = """
      <div id='vis'></div>
    """
    display(HTML(vis_html))
    __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    vis_js = open(os.path.join(__location__, 'model_view_gpt2.js')).read()
    attn_data = get_attention_gpt2(model, tokenizer, text)
    att_json = json.dumps(attn_data)
    display(Javascript('window.attention = %s' % att_json))
    display(Javascript(vis_js))

コード例 #3

ファイル: head_view_gpt2.py プロジェクト: huangpeng1126/bertviz

def show(model, tokenizer, text):
    vis_html = """
      <span style="user-select:none">
        Layer: <select id="layer"></select>
      </span>
      <div id='vis'></div>
    """
    display(HTML(vis_html))
    __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    vis_js = open(os.path.join(__location__, 'head_view_gpt2.js')).read()
    attn_data = get_attention_gpt2(model, tokenizer, text)
    att_json = json.dumps(attn_data)
    display(Javascript('window.attention = %s' % att_json))
    display(Javascript(vis_js))

コード例 #4

ファイル: test_attention.py プロジェクト: huangpeng1126/bertviz

 def test_gpt2_attn(self):
     model = GPT2Model.from_pretrained('gpt2')
     tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
     text = 'Bert is a yellow muppet character'
     attn_data = get_attention_gpt2(model,
                                    tokenizer,
                                    text,
                                    include_queries_and_keys=False)
     tokens = ['B', 'ert', 'is', 'a', 'yellow', 'm', 'uppet', 'character']
     self.assertEqual(attn_data['left_text'], tokens)
     self.assertEqual(attn_data['right_text'], tokens)
     seq_len = len(tokens)
     layer = 0
     head = 0
     att_matrix = attn_data['attn'][layer][head]
     for i in range(seq_len):
         for j in range(seq_len):
             if i >= j:
                 self.assertNotEqual(att_matrix[i][j], 0)
             else:
                 self.assertEqual(att_matrix[i][j], 0)
         sum_probs = sum(att_matrix[i])
         self.assertAlmostEqual(sum_probs, 1, 4)

コード例 #5

def show(model, tokenizer, text):
    vis_html = """
      <span style="user-select:none">
        Layer: <select id="layer"></select>
        Head: <select id="att_head"></select>
      </span>
      <div id='vis'></div>
    """
    display(HTML(vis_html))
    __location__ = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    vis_js = open(os.path.join(__location__, 'neuron_view.js')).read()
    attn_data = get_attention_gpt2(model,
                                   tokenizer,
                                   text,
                                   include_queries_and_keys=True)
    params = {
        'attention': attn_data,
        'default_filter': "all",
        'bidirectional': False
    }
    display(Javascript('window.params = %s' % json.dumps(params)))
    display(Javascript(vis_js))