def test_load_empty_memory(self): current_path = os.path.dirname(os.path.abspath(__file__)) checkpoint_path = os.path.join(current_path, 'test_checkpoint_empty') model = load_trained_model_from_checkpoint( config_path=os.path.join(checkpoint_path, 'xlnet_config.json'), checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'), batch_size=2, memory_len=5, target_len=5, in_train_phase=False, mask_index=0, attention_type=ATTENTION_TYPE_UNI, ) set_custom_objects() model_path = os.path.join(tempfile.gettempdir(), 'test_xlnet_%f.h5' % np.random.random()) model.save(model_path) model = keras.models.load_model(model_path) model.summary() def _load_numpy(name): return np.load(os.path.join(checkpoint_path, name + '.npy')) input_ids = _load_numpy('input_ids') seg_ids = _load_numpy('seg_ids') tune_output = _load_numpy('empty_output') inputs = [input_ids, seg_ids, np.zeros((2, 1))] output = model.predict_on_batch(inputs) self.assertTrue(np.allclose(tune_output, output, atol=1e-6))
def test_build_not_training(self): model = build_xlnet( units=6, training=False, num_token=31, num_block=2, num_head=2, hidden_dim=12, batch_size=2, memory_len=5, target_len=5, dropout=0.1, attention_dropout=0.1, attention_type=ATTENTION_TYPE_BI, ) set_custom_objects() model_path = os.path.join(tempfile.gettempdir(), 'test_xlnet_%f.h5' % np.random.random()) model.save(model_path) model = keras.models.load_model(model_path) model.summary() try: current_path = os.path.dirname(os.path.abspath(__file__)) visual_path = os.path.join(current_path, 'test_build_not_training.jpg') keras.utils.vis_utils.plot_model(model, visual_path, show_shapes=True) except Exception as e: pass
def test_load_training(self): current_path = os.path.dirname(os.path.abspath(__file__)) checkpoint_path = os.path.join(current_path, 'test_checkpoint_pre') model = load_trained_model_from_checkpoint( config_path=os.path.join(checkpoint_path, 'xlnet_config.json'), checkpoint_path=os.path.join(checkpoint_path, 'xlnet_model.ckpt'), batch_size=2, memory_len=5, target_len=5, in_train_phase=True, mask_index=0, attention_type=ATTENTION_TYPE_UNI, ) set_custom_objects() model_path = os.path.join(tempfile.gettempdir(), 'test_xlnet_%f.h5' % np.random.random()) model.save(model_path) model = keras.models.load_model(model_path) model.summary() def _load_numpy(name): return np.load(os.path.join(checkpoint_path, name + '.npy')) input_ids = _load_numpy('input_ids') seg_ids = _load_numpy('seg_ids') input_mask = _load_numpy('input_mask') mems_0 = _load_numpy('mems_0') mems_1 = _load_numpy('mems_1') pre_output = _load_numpy('pre_output') pre_new_mems_0 = _load_numpy('pre_new_mems_0') pre_new_mems_1 = _load_numpy('pre_new_mems_1') inputs = [input_ids, seg_ids, np.ones((2, 1)) * 5, input_mask] self._update_memory(model, 'Memory-0', mems_0) self._update_memory(model, 'Memory-1', mems_1) output = model.predict_on_batch(inputs) self.assertTrue( np.allclose(pre_new_mems_0, self._get_memory(model, 'Memory-0', 5), atol=1e-6)) self.assertTrue( np.allclose(pre_new_mems_1, self._get_memory(model, 'Memory-1', 5), atol=1e-6)) self.assertTrue(np.allclose(pre_output, output, atol=1e-6))
def build(self): from keras_xlnet import load_trained_model_from_checkpoint, set_custom_objects from keras_xlnet import Tokenizer, ATTENTION_TYPE_BI, ATTENTION_TYPE_UNI self.embedding_type = 'xlnet' self.checkpoint_path = os.path.join(self.corpus_path, 'xlnet_model.ckpt') self.config_path = os.path.join(self.corpus_path, 'xlnet_config.json') self.spiece_model = os.path.join(self.corpus_path, 'spiece.model') self.attention_type = self.xlnet_embed.get('attention_type', 'bi') # or 'uni' self.attention_type = ATTENTION_TYPE_BI if self.attention_type == 'bi' else ATTENTION_TYPE_UNI self.memory_len = self.xlnet_embed.get('memory_len', 0) self.target_len = self.xlnet_embed.get('target_len', 5) print('load xlnet model start!') # 模型加载 model = load_trained_model_from_checkpoint( checkpoint_path=self.checkpoint_path, attention_type=self.attention_type, in_train_phase=self.trainable, config_path=self.config_path, memory_len=self.memory_len, target_len=self.target_len, batch_size=self.batch_size, mask_index=0) # set_custom_objects() # 字典加载 self.tokenizer = Tokenizer(self.spiece_model) # debug时候查看layers self.model_layers = model.layers len_layers = self.model_layers.__len__() print(len_layers) layer_real = [i for i in range(25)] + [-i for i in range(25)] # 简要判别一下 self.layer_indexes = [ i if i in layer_real else -2 for i in self.layer_indexes ] len_couche = int((len_layers - 6) / 10) # 一共246个layer # 每层10个layer(MultiHeadAttention,Dropout,Add,LayerNormalization),第一是9个layer的输入和embedding层 # 一共24层 layer_dict = [] layer_0 = 7 for i in range(len_couche): layer_0 = layer_0 + 10 layer_dict.append(layer_0) layer_dict.append(247) # 测试 get_output_at # def get_number(index): # try: # model_node = model.get_output_at(node_index=index) # gg = 0 # except: # print('node index wrong!') # print(index) # list_index = [i for i in range(25)] + [-i for i in range(25)] # for li in list_index: # get_number(li) # 输出它本身 if len(self.layer_indexes) == 0: encoder_layer = model.output # 分类如果只有一层,取得不正确的话就取倒数第二层 elif len(self.layer_indexes) == 1: if self.layer_indexes[0] in layer_real: encoder_layer = model.get_layer( index=layer_dict[self.layer_indexes[0]]).get_output_at( node_index=0) else: encoder_layer = model.get_layer( index=layer_dict[-1]).get_output_at(node_index=0) # 否则遍历需要取的层,把所有层的weight取出来并加起来shape:768*层数 else: # layer_indexes must be [0, 1, 2,3,......24] all_layers = [ model.get_layer(index=layer_dict[lay]).get_output_at( node_index=0) if lay in layer_real else model.get_layer( index=layer_dict[-1]).get_output_at( node_index=0) # 如果给出不正确,就默认输出倒数第一层 for lay in self.layer_indexes ] print(self.layer_indexes) print(all_layers) all_layers_select = [] for all_layers_one in all_layers: all_layers_select.append(all_layers_one) encoder_layer = Add()(all_layers_select) print(encoder_layer.shape) # def xlnet_concat(x): # x_concat = K.concatenate(x, axis=1) # return x_concat # encoder_layer = Lambda(xlnet_concat, name='xlnet_concat')(all_layers) self.output = NonMaskingLayer()(encoder_layer) self.input = model.inputs self.model = Model(self.input, self.output) print("load KerasXlnetEmbedding end") model.summary(132) self.embedding_size = self.model.output_shape[-1] self.vocab_size = len(self.tokenizer.sp)
def build(self): from keras_xlnet import load_trained_model_from_checkpoint, set_custom_objects from keras_xlnet import Tokenizer, ATTENTION_TYPE_BI, ATTENTION_TYPE_UNI self.embedding_type = 'xlnet' self.checkpoint_path = os.path.join(self.corpus_path, 'xlnet_model.ckpt') self.config_path = os.path.join(self.corpus_path, 'xlnet_config.json') self.spiece_model = os.path.join(self.corpus_path, 'spiece.model') self.attention_type = self.xlnet_embed.get('attention_type', 'bi') # or 'uni' self.attention_type = ATTENTION_TYPE_BI if self.attention_type == 'bi' else ATTENTION_TYPE_UNI self.memory_len = self.xlnet_embed.get('memory_len', 0) self.target_len = self.xlnet_embed.get('target_len', 5) print('load xlnet model start!') # 模型加载 model = load_trained_model_from_checkpoint(checkpoint_path=self.checkpoint_path, attention_type=self.attention_type, in_train_phase=self.trainable, config_path=self.config_path, memory_len=self.memory_len, target_len=self.target_len, batch_size=self.batch_size, mask_index=0) # set_custom_objects() self.build_config(self.config_path) # 字典加载 self.tokenizer = Tokenizer(self.spiece_model) # # debug时候查看layers # self.model_layers = model.layers # len_layers = self.model_layers.__len__() # print(len_layers) num_hidden_layers = self.configs.get("n_layer", 12) layer_real = [i for i in range(num_hidden_layers)] + [-i for i in range(num_hidden_layers)] # 简要判别一下 self.layer_indexes = [i if i in layer_real else -2 for i in self.layer_indexes] output_layer = "FeedForward-Normal-{0}" layer_dict = [model.get_layer(output_layer.format(i + 1)).get_output_at(node_index=0) for i in range(num_hidden_layers)] # 输出它本身 if len(self.layer_indexes) == 0: encoder_layer = model.output # 分类如果只有一层,取得不正确的话就取倒数第二层 elif len(self.layer_indexes) == 1: if self.layer_indexes[0] in layer_real: encoder_layer = layer_dict[self.layer_indexes[0]] else: encoder_layer = layer_dict[-1] # 否则遍历需要取的层,把所有层的weight取出来并加起来shape:768*层数 else: # layer_indexes must be [0, 1, 2,3,......24] all_layers = [layer_dict[lay] if lay in layer_real else layer_dict[-1] # 如果给出不正确,就默认输出倒数第一层 for lay in self.layer_indexes] print(self.layer_indexes) print(all_layers) all_layers_select = [] for all_layers_one in all_layers: all_layers_select.append(all_layers_one) encoder_layer = Add()(all_layers_select) print(encoder_layer.shape) # def xlnet_concat(x): # x_concat = K.concatenate(x, axis=1) # return x_concat # encoder_layer = Lambda(xlnet_concat, name='xlnet_concat')(all_layers) self.output = NonMaskingLayer()(encoder_layer) self.input = model.inputs self.model = Model(self.input, self.output) print("load KerasXlnetEmbedding end") model.summary(132) self.embedding_size = self.model.output_shape[-1] self.vocab_size = len(self.tokenizer.sp)