def main():
try:
consumer = KafkaConsumer(KAFKA_TOPIC,
bootstrap_servers=KAFKA_BOOTSTRAP_SERVERS,
auto_offset_reset='earliest',
enable_auto_commit=False,
group_id=KAFKA_CONSUMER_GROUP)
session = tf.Session(graph=tf.Graph())
with session.graph.as_default():
keras.backend.set_session(session)
check_model_files()
model = load_model(MODEL_JSON_PATH, MODEL_WEIGHTS_PATH)
logger.info("Start listing")
for msg in consumer:
logger.info(msg)
id = msg.value.decode()
logger.info("Start processing id : {}".format(id))
process_msg(id, model)
consumer.commit()
logger.info("Finish processing id : {}".format(id))
finally:
logger.info("closing db")
db_session.remove()
def answer(dataset = []):
model, (words, labels, training, output) = load_model()
print("type (q) to stop!")
while True:
sentence = input("You: ")
if sentence.lower() == "q":
break
answ = do_answer(sentence, dataset=dataset)
print(answ)
def run_window(dataset=[]):
model, (words, labels, training, output) = load_model()
main_window = tk.Tk(screenName='MainScreen')
main_window.title('Chat with Bot')
main_window.geometry("950x600+300+300")
menu = tk.Menu(master=main_window)
main_window.config(menu=menu, background='#75bdbd')
file_menu = tk.Menu(master=menu)
menu.add_cascade(label='Chatbot', menu=file_menu)
file_menu.add_command(label='About')
menu.add_separator()
file_menu.add_command(label='Exit', command=main_window.quit)
label_1 = tk.Label(master=main_window, text='Pesanmu : ')
label_1.grid(row=0)
entry_1_text = tk.StringVar()
entry_1 = tk.Entry(master=main_window, width=80, textvariable=entry_1_text)
entry_1.grid(row=0, column=1)
scrollbar = tk.Scrollbar(master=main_window, orient='vertical')
scrollbar.grid(row=3, column=2)
list_box = tk.Listbox(master=main_window,
width=80,
yscrollcommand=scrollbar.set)
list_box.grid(row=3, column=1)
def btn_1_click():
if entry_1_text.get() != '':
sentence = entry_1_text.get()
answ = do_answer(sentence, dataset=dataset)
list_box.insert(tk.END, 'Aku: {}'.format(sentence))
list_box.insert(tk.END, 'Bot: {}'.format(answ))
entry_1_text.set('')
btn_1 = tk.Button(master=main_window,
text='Send',
width=10,
command=btn_1_click)
btn_1.config(background='#0fa4d6')
btn_1.grid(row=0, column=3)
main_window.mainloop()
def predict_prob(model_file: click.Path,
test_data: click.Path,
k: int,
t: float,
batch_size: int,
stat: bool,
show_data: bool,
):
"""predict most likely labels with probabilities"""
# metadata
metadata_file = f"{model_file}.meta.yml"
metadata = Metadata.load(metadata_file)
# load model
model = load_model(model_file)
# test data
dataset_test = read(test_data)
dataset_test = Dataset(
metadata.task,
metadata.labels,
metadata.chars,
dataset_test.samples,
)
if stat:
cowsay.cow(f"Test data stat:\n {utils.stat(dataset_test)}")
# prediction
pred = model.predict_generator(
BatchSequence(dataset_test, batch_size, metadata.params['maxlen']),
workers=1,
use_multiprocessing=True)
indices = numpy.argsort(-pred)[:, :k]
for i, ind in enumerate(indices):
if metadata.task == Task.binary:
pred_label = metadata.labels[0 if pred[i, 0] < 0.5 else 1]
pred_prob = utils.float4(max(pred[i, 0], 1.0 - pred[i, 0]))
pred_msg = f"{pred_label} {pred_prob}"
else:
pred_labels = [metadata.labels[j] for j in ind if pred[i, j] > t]
pred_probs = [utils.float4(pred[i, j]) for j in ind if pred[i, j] > t]
pred_msg = ' '.join(str(x) for pair in zip(pred_labels, pred_probs) for x in pair)
if show_data:
print(pred_msg, dataset_test.samples[i].data)
else:
print(pred_msg)
def print_sentence_vectors(
model_file: str,
data_file: str,
batch_size: int,
):
"""print sentence vectors given a trained model"""
# metadata
metadata_file = f"{model_file}.meta.yml"
metadata = Metadata.load(metadata_file)
# load model
model = load_model(model_file)
feature_model = Model(inputs=model.input,
outputs=model.get_layer('dense_1').output)
# data
dataset = read(data_file)
pred = feature_model.predict_generator(
BatchSequence(dataset, batch_size, metadata.params['maxlen']),
workers=1,
use_multiprocessing=True)
for v in pred:
print(' '.join(str(utils.float4(x)) for x in v))
def test(
model_file: str,
test_data: str,
k: int,
t: float,
batch_size: int,
stat: bool,
):
"""evaluate a supervised classifier"""
# metadata
metadata_file = f"{model_file}.meta.yml"
metadata = Metadata.load(metadata_file)
# load model
model = load_model(model_file)
# test data
dataset_test = read(test_data, remove_no_labels=True)
dataset_test = Dataset(
metadata.task,
metadata.labels,
metadata.chars,
dataset_test.samples,
)
if stat:
cowsay.cow(f"Test data stat:\n {utils.stat(dataset_test)}")
# prediction
pred = model.predict_generator(
BatchSequence(dataset_test, batch_size, metadata.params['maxlen']),
workers=1,
use_multiprocessing=True)
indices = numpy.argsort(-pred)[:, :k]
n_labels = len(dataset_test.labels)
confusion_matrix = [
[0] * n_labels
for _ in range(n_labels)
]
if metadata.task == Task.binary:
for i in range(len(pred)):
y_pred = 1 if pred[i, 0] > 0.5 else 0
label_true = dataset_test.samples[i].labels[0]
y_true = metadata.labels.index(label_true)
confusion_matrix[y_true][y_pred] += 1
n_sum = sum(confusion_matrix[i][j]
for i in range(n_labels)
for j in range(n_labels))
n_acc = sum(confusion_matrix[i][i] for i in range(n_labels))
print(f"Acc: {n_acc / n_sum:.4f}")
performance = utils.labels_performance(confusion_matrix)
for i, perf in enumerate(performance):
print(metadata.labels[i])
print(f"Recall: {perf['recall']:.4f}")
print(f"Prec: {perf['prec']:.4f}")
print(f"F1: {perf['f1']:.4f}")
else:
n_sum = 0
n_acc = 0
for i, ind in enumerate(indices):
preds = [j for j in ind if pred[i, j] > t]
truth = [metadata.labels.index(label) for label in dataset_test.samples[i].labels]
n_sum += 1
if all(j in preds for j in truth):
n_acc += 1
for y_pred in preds:
for y_true in truth:
confusion_matrix[y_true][y_pred] += 1
print(f"Acc@{k}: {n_acc / n_sum:.4f}")
performance = utils.labels_performance(confusion_matrix)
for i, perf in enumerate(performance):
print(metadata.labels[i])
print(f"- Recall: {perf['recall']:.4f}")
print(f"- Prec: {perf['prec']:.4f}")
print(f"- F1: {perf['f1']:.4f}")