def load(self):
load_folder_path = self.save_folder_path
with open(os.path.join(load_folder_path, 'params.json'), 'r') as json_file:
model_params = json.load(json_file)
slots_num = model_params['slots_num']
intents_num = model_params['intents_num']
bert_model_hub_path = model_params['bert_hub_path']
num_bert_fine_tune_layers = model_params['num_bert_fine_tune_layers']
is_bert = model_params['is_bert']
if 'is_crf' in model_params:
is_crf = model_params['is_crf']
else:
is_crf = False
self.bert_vectorizer = BERTVectorizer(self.sess, is_bert, bert_model_hub_path)
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'), 'rb') as handle:
self.tags_vectorizer = pickle.load(handle)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
self.intents_label_encoder = pickle.load(handle)
intents_num = len(self.intents_label_encoder.classes_)
if is_crf:
self.model = JointBertCRFModel.load(load_folder_path, self.sess)
else:
self.model = JointBertModel.load(load_folder_path, self.sess)
def initialize():
global graph
graph = tf.get_default_graph()
global sess
sess = tf.compat.v1.Session()
set_session(sess)
global bert_vectorizer
bert_vectorizer = BERTVectorizer(sess, is_bert, bert_model_hub_path)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
global slots_num
global tags_vectorizer
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
global intents_num
global intents_label_encoder
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)
global model
try:
model = JointBertModel.load(load_folder_path, sess)
except OSError:
model = JointBertCRFModel.load(load_folder_path, sess)
def initialize():
global bert_vectorizer
bert_vectorizer = BERTVectorizer(
True, "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1")
# loading models
global model
model = JointBertModel1.load("./models")
def initialize():
global bert_vectorizer
bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
global slots_num
global tags_vectorizer
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
global intents_num
global intents_label_encoder
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)
global model
model = JointBertModel.load(load_folder_path)
parser.add_argument('--batch',
'-bs',
help='Batch size',
type=int,
default=128,
required=False)
args = parser.parse_args()
load_folder_path = args.model
data_folder_path = args.data
batch_size = args.batch
sess = tf.compat.v1.Session()
bert_model_hub_path = 'https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1'
bert_vectorizer = BERTVectorizer(sess, bert_model_hub_path)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)
"6": "Open_ticket",
"7": "Request_PTO",
"8": "Reset_password",
"9": "Update_ticket",
"10": "affirm",
"11": "deny",
"12": "goodbye",
"13": "greet",
"14": "high_intent",
"15": "live_agent",
"16": "low_intent",
"17": "medium_intent",
"18": "stop",
"19": "welcome"
}
bert_vectorizer = BERTVectorizer(
True, "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1")
model = JointBertModel1.load("./models")
import time
app = Flask(__name__)
@app.route('/intent', methods=['POST'])
def predict():
data = request.get_json()["utterance"]
print("---------------")
print(data)
a = time.time()
#tokens = utterance.split()
#print(utterance)
input_ids, input_mask, segment_ids, data_sequence_lengths = bert_vectorizer.transform(
[data])
def train(self, train_data_dir, valid_data_dir):
self.train_data_folder_path = train_data_dir
self.val_data_folder_path = valid_data_dir
if self.type_ == 'bert':
bert_model_hub_path = 'https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1'
is_bert = True
else:
bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
is_bert = False
#read data
train_text_arr, train_tags_arr, train_intents = Reader.read(self.train_data_folder_path)
val_text_arr, val_tags_arr, val_intents = Reader.read(self.val_data_folder_path)
#vectorise data
self.bert_vectorizer = BERTVectorizer(self.sess, is_bert, bert_model_hub_path)
train_input_ids, train_input_mask, train_segment_ids, train_valid_positions, train_sequence_lengths = self.bert_vectorizer.transform(train_text_arr)
val_input_ids, val_input_mask, val_segment_ids, val_valid_positions, val_sequence_lengths = self.bert_vectorizer.transform(val_text_arr)
#vectorise tags
if self.use_crf: # with crf
self.tags_vectorizer = TagsVectorizer()
self.tags_vectorizer.fit(train_tags_arr)
train_tags = self.tags_vectorizer.transform(train_tags_arr, train_valid_positions)
train_tags = tf.keras.utils.to_categorical(train_tags)
val_tags = self.tags_vectorizer.transform(val_tags_arr, val_valid_positions)
val_tags = tf.keras.utils.to_categorical(val_tags)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
else: # without crf
self.tags_vectorizer = TagsVectorizer()
self.tags_vectorizer.fit(train_tags_arr)
train_tags = self.tags_vectorizer.transform(train_tags_arr, train_valid_positions)
val_tags = self.tags_vectorizer.transform(val_tags_arr, val_valid_positions)
slots_num = len(self.tags_vectorizer.label_encoder.classes_)
#encode labels
self.intents_label_encoder = LabelEncoder()
train_intents = self.intents_label_encoder.fit_transform(train_intents).astype(np.int32)
val_intents = self.intents_label_encoder.transform(val_intents).astype(np.int32)
intents_num = len(self.intents_label_encoder.classes_)
#train
if self.use_crf: # with crf
self.model = JointBertCRFModel(slots_num, intents_num, bert_model_hub_path, self.sess, num_bert_fine_tune_layers=10, is_bert=is_bert, is_crf=True, learning_rate=self.learning_rate)
self.model.fit([train_input_ids, train_input_mask, train_segment_ids, train_valid_positions, train_sequence_lengths], [train_tags, train_intents],
validation_data=([val_input_ids, val_input_mask, val_segment_ids, val_valid_positions, val_sequence_lengths], [val_tags, val_intents]),
epochs=self.epochs, batch_size=self.batch_size)
else: # without crf
self.model = JointBertModel(slots_num, intents_num, bert_model_hub_path, self.sess, num_bert_fine_tune_layers=10, is_bert=is_bert, is_crf=False, learning_rate=self.learning_rate)
self.model.fit([train_input_ids, train_input_mask, train_segment_ids, train_valid_positions], [train_tags, train_intents],
validation_data=([val_input_ids, val_input_mask, val_segment_ids, val_valid_positions], [val_tags, val_intents]),
epochs=self.epochs, batch_size=self.batch_size)
#save
if not os.path.exists(self.save_folder_path):
os.makedirs(self.save_folder_path)
self.model.save(self.save_folder_path)
with open(os.path.join(self.save_folder_path, 'tags_vectorizer.pkl'), 'wb') as handle:
pickle.dump(self.tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(self.save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle:
pickle.dump(self.intents_label_encoder, handle, protocol=pickle.HIGHEST_PROTOCOL)
args = parser.parse_args()
load_folder_path = args.model
data_folder_path = args.data
batch_size = args.batch
type_ = args.type
if type_ == 'bert':
bert_model_hub_path = "https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1"
is_bert = True
elif type_ == 'albert':
bert_model_hub_path = "https://tfhub.dev/tensorflow/albert_en_base/1"
is_bert = False
else:
raise ValueError('type must be one of these values: %s' % str(VALID_TYPES))
bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)
# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
print('Folder `%s` not exist' % load_folder_path)
with open(os.path.join(load_folder_path, 'tags_vectorizer.pkl'),
'rb') as handle:
tags_vectorizer = pickle.load(handle)
slots_num = len(tags_vectorizer.label_encoder.classes_)
with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'),
'rb') as handle:
intents_label_encoder = pickle.load(handle)
intents_num = len(intents_label_encoder.classes_)