def load(self):
load_folder_path = self.save_folder_path
with open(os.path.join(load_folder_path, 'params.json'), 'r') as json_file:
model_params = json.load(json_file)
slots_num = model_params['slots_num']
intents_num = model_params['intents_num']
bert_model_hub_path = model_params['bert_hub_path']
num_bert_fine_tune_layers = model_params['num_bert_fine_tune_layers']
is_bert = model_params['is_bert']
if 'is_crf' in model_params:
is_crf = model_params['is_crf']
else:
is_crf = False
self.bert_vectorizer = BERTVectorizer(self.sess, is_bert, bert_model_hub_path)
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'), 'rb') as handle:
self.tags_vectorizer = pickle.load(handle)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
self.intents_label_encoder = pickle.load(handle)
intents_num = len(self.intents_label_encoder.classes_)
if is_crf:
self.model = JointBertCRFModel.load(load_folder_path, self.sess)
else:
self.model = JointBertModel.load(load_folder_path, self.sess)
def initialize():
global graph
graph = tf.get_default_graph()
global sess
sess = tf.compat.v1.Session()
set_session(sess)
global bert_vectorizer
bert_vectorizer = BERTVectorizer(sess, is_bert, bert_model_hub_path)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
global slots_num
global tags_vectorizer
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
global intents_num
global intents_label_encoder
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)
global model
try:
model = JointBertModel.load(load_folder_path, sess)
except OSError:
model = JointBertCRFModel.load(load_folder_path, sess)
#enc = OneHotEncoder(handle_unknown='ignore', sparse=False)
#enc.fit(train_tags)
train_tags = tf.keras.utils.to_categorical(train_tags)
val_tags = tags_vectorizer.transform(val_tags_arr, val_valid_positions)
val_tags = tf.keras.utils.to_categorical(val_tags)
slots_num = len(tags_vectorizer.label_encoder.classes_)
intents_label_encoder = LabelEncoder()
train_intents = intents_label_encoder.fit_transform(train_intents).astype(np.int32)
val_intents = intents_label_encoder.transform(val_intents).astype(np.int32)
intents_num = len(intents_label_encoder.classes_)
model = JointBertCRFModel(slots_num, intents_num, sess, num_bert_fine_tune_layers=10)
model.fit([train_input_ids, train_input_mask, train_segment_ids, train_valid_positions, train_sequence_lengths], [train_tags, train_intents],
validation_data=([val_input_ids, val_input_mask, val_segment_ids, val_valid_positions, val_sequence_lengths], [val_tags, val_intents]),
epochs=epochs, batch_size=batch_size)
### saving
print('Saving ..')
if not os.path.exists(save_folder_path):
os.makedirs(save_folder_path)
print('Folder `%s` created' % save_folder_path)
model.save(save_folder_path)
with open(os.path.join(save_folder_path, 'tags_vectorizer.pkl'), 'wb') as handle:
pickle.dump(tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle:
def train(self, train_data_dir, valid_data_dir):
self.train_data_folder_path = train_data_dir
self.val_data_folder_path = valid_data_dir
if self.type_ == 'bert':
bert_model_hub_path = 'https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1'
is_bert = True
else:
bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
is_bert = False
#read data
train_text_arr, train_tags_arr, train_intents = Reader.read(self.train_data_folder_path)
val_text_arr, val_tags_arr, val_intents = Reader.read(self.val_data_folder_path)
#vectorise data
self.bert_vectorizer = BERTVectorizer(self.sess, is_bert, bert_model_hub_path)
train_input_ids, train_input_mask, train_segment_ids, train_valid_positions, train_sequence_lengths = self.bert_vectorizer.transform(train_text_arr)
val_input_ids, val_input_mask, val_segment_ids, val_valid_positions, val_sequence_lengths = self.bert_vectorizer.transform(val_text_arr)
#vectorise tags
if self.use_crf: # with crf
self.tags_vectorizer = TagsVectorizer()
self.tags_vectorizer.fit(train_tags_arr)
train_tags = self.tags_vectorizer.transform(train_tags_arr, train_valid_positions)
train_tags = tf.keras.utils.to_categorical(train_tags)
val_tags = self.tags_vectorizer.transform(val_tags_arr, val_valid_positions)
val_tags = tf.keras.utils.to_categorical(val_tags)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
else: # without crf
self.tags_vectorizer = TagsVectorizer()
self.tags_vectorizer.fit(train_tags_arr)
train_tags = self.tags_vectorizer.transform(train_tags_arr, train_valid_positions)
val_tags = self.tags_vectorizer.transform(val_tags_arr, val_valid_positions)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
#encode labels
self.intents_label_encoder = LabelEncoder()
train_intents = self.intents_label_encoder.fit_transform(train_intents).astype(np.int32)
val_intents = self.intents_label_encoder.transform(val_intents).astype(np.int32)
intents_num = len(self.intents_label_encoder.classes_)
#train
if self.use_crf: # with crf
self.model = JointBertCRFModel(slots_num, intents_num, bert_model_hub_path, self.sess, num_bert_fine_tune_layers=10, is_bert=is_bert, is_crf=True, learning_rate=self.learning_rate)
self.model.fit([train_input_ids, train_input_mask, train_segment_ids, train_valid_positions, train_sequence_lengths], [train_tags, train_intents],
validation_data=([val_input_ids, val_input_mask, val_segment_ids, val_valid_positions, val_sequence_lengths], [val_tags, val_intents]),
epochs=self.epochs, batch_size=self.batch_size)
else: # without crf
self.model = JointBertModel(slots_num, intents_num, bert_model_hub_path, self.sess, num_bert_fine_tune_layers=10, is_bert=is_bert, is_crf=False, learning_rate=self.learning_rate)
self.model.fit([train_input_ids, train_input_mask, train_segment_ids, train_valid_positions], [train_tags, train_intents],
validation_data=([val_input_ids, val_input_mask, val_segment_ids, val_valid_positions], [val_tags, val_intents]),
epochs=self.epochs, batch_size=self.batch_size)
#save
if not os.path.exists(self.save_folder_path):
os.makedirs(self.save_folder_path)
self.model.save(self.save_folder_path)
with open(os.path.join(self.save_folder_path, 'tags_vectorizer.pkl'), 'wb') as handle:
pickle.dump(self.tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(self.save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle:
pickle.dump(self.intents_label_encoder, handle, protocol=pickle.HIGHEST_PROTOCOL)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)
model = JointBertCRFModel.load(load_folder_path)
data_text_arr, data_tags_arr, data_intents = Reader.read(data_folder_path)
data_input_ids, data_input_mask, data_segment_ids, data_valid_positions, data_sequence_lengths = bert_vectorizer.transform(
data_text_arr)
def get_results(input_ids, input_mask, segment_ids, valid_positions,
sequence_lengths, tags_arr, intents, tags_vectorizer,
intents_label_encoder):
predicted_tags, predicted_intents = model.predict_slots_intent(
[
input_ids, input_mask, segment_ids, valid_positions,
sequence_lengths
],
tags_vectorizer,