def connect_serial(self):
if (self.ip != '' and self.ip != None) and (
self.port != '' and self.port != None) and (self.connect
== False):
addr = self.ip + ":" + str(self.port)
try:
self.serial = serial.serial_for_url("socket://" + addr +
"/logging=debug")
self.connect = True
return True
except Exception as e:
log('error', "Device.py :" + str(e))
self.close()
return False
elif (self.com != '') and (self.connect == False):
try:
self.serial = serial.Serial(self.com)
self.connect = True
return True
except Exception as e:
log('error', "Device.py :" + str(e))
self.close()
return False
elif (self.connect == True):
return True
else:
return False
def __init__(self):
config = configparser.ConfigParser()
try:
config.read('config.env')
self.API_URL = config['CONFIG']['API_URL']
self.LOG_FILE = config['CONFIG']['LOG_FILE']
self.LOG_LEVEL = config['CONFIG']['LOG_LEVEL']
self.WEIGHT_PORT = config['FOOD']['WEIGHT_PORT']
self.WEIGHT_IP = config['FOOD']['WEIGHT_IP']
self.WEIGHT_COM = config['FOOD']['WEIGHT_COM']
self.FOOD_RFID_IP = config['FOOD']['RFID_IP']
self.FOOD_RFID_PORT = config['FOOD']['RFID_PORT']
self.FOOD_RFID_COM = config['FOOD']['RFID_COM']
self.WATER_RFID_IP = config['WATER']['RFID_IP']
self.WATER_RFID_PORT = config['WATER']['RFID_PORT']
self.WATER_RFID_COM = config['WATER']['RFID_COM']
self.WATER_IP = config['WATER']['WATER_IP']
self.WATER_PORT = config['WATER']['WATER_PORT']
self.WATER_COM = config['WATER']['WATER_COM']
log('debug', 'start config success')
except:
log('error', 'please init your env file')
print("please init your env file")
pass
def food_device_listen():
try:
food_device.listen()
except Exception as e:
print("FOOD DEVICE ERROR:")
print(e)
log('error', "Food Device Listen Error:" + str(e))
pass
def food_rfid_listen():
try:
food_rfid.listen()
except Exception as e:
print("FOOD RFID ERROR:")
print(e)
log('error', "FOOD RFID Listen Error:" + str(e))
pass
def water_rfid_listen():
try:
water_rfid.listen()
except Exception as e:
print("WATER RFID ERROR:")
print(e)
log('error', "WATER RFID Listen Error:" + str(e))
pass
def model_params():
log('\n\n' + 'MODEL PARAMETERS :' + '\n\n' + 'EMBEDDING_SIZE = ' +
str(Config.EMBEDDING_SIZE) + '\n' + 'DROPOUT = ' +
str(Config.DROPOUT) + '\n' + 'N_EPOCHS = ' + str(Config.N_EPOCHS) +
'\n' + 'LOSS = ' + str(Config.LOSS) + '\n' + 'OPTIMIZER = ' +
str(Config.OPTIMIZER) + '\n' + 'MODEL = ' + str(Config.MODEL) + '\n' +
'DATA_FILE = ' + str(Config.DATA_FILE) + '\n' + 'PORT = ' +
str(Config.PORT) + '\n' + 'WORD_EMBEDDINGS = ' +
str(Config.WORD_EMBEDDINGS) + '\n' + '\n\n')
def get_value(self):
if self.connect_serial():
r = self.write(self.READ_COMMAND, 2.8)
# print(r)
r = r[30:]
r = r[:26]
bval = r[1:10]
bval.reverse()
sval = ""
ival = 0
bop = r[-2:]
sop = ""
iop = 0
xval = r[13:]
xval = xval[1:10]
xval.reverse()
sxval = ""
ixval = 0
xop = r[-2:]
sxop = ""
ixop = 0
for i in bval:
i = i.decode()
sval += i
for o in bop:
o = o.decode()
sop += o
iop = int(sop)
ival = float(sval) * (10 ** iop)
for x in xval:
x = x.decode()
sxval += x
for xo in xop:
xo = xo.decode()
sxop += xo
ixop = int(sxop)
ixval = float(sxval) * (10 ** ixop)
self.count += 1
if self.count > 5:
self.close()
return ival - ixval
else:
log('error', "WATER_DEVICE: not connect device")
print("WATER_DEVICE: not connect device")
return None
def checkDevice(self):
if (self.food_rfid.connect_serial()
and self.food_device.connect_serial()
and self.water_rfid.connect_serial()
and self.water_device.connect_serial()):
self.food_rfid.close()
self.food_device.close()
self.water_rfid.close()
self.water_device.close()
else:
log('error', "can't connect device with serials!")
print("Connect Device Error!")
def __init__(self):
super(MainWindow, self).__init__()
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.food_model= QtGui.QStandardItemModel(10,4)
self.food_model.setHorizontalHeaderLabels(['UID','IN_TIME','OUT_TIME','VAL'])
self.ui.tableView.setModel(self.food_model)
self.water_model= QtGui.QStandardItemModel(10,4)
self.water_model.setHorizontalHeaderLabels(['UID','IN_TIME','OUT_TIME','VAL'])
self.ui.tableView_2.setModel(self.water_model)
config = Config.get_instance()
self.weight_device = Weight_Device(
ip=config.WEIGHT_IP, port=config.WEIGHT_PORT, com=config.WEIGHT_COM)
self.food_rfid_device = RFID(
ip=config.FOOD_RFID_IP, port=config.FOOD_RFID_PORT, com=config.FOOD_RFID_COM, name="FOOD_RFID")
self.water_rfid_device = RFID(
ip=config.WATER_RFID_IP, port=config.WATER_RFID_PORT, com=config.WATER_RFID_COM, name="WARTER_RFID")
print("WATER RFID Connect status:" + str(self.water_rfid_device.connect_serial()))
if (self.weight_device.connect_serial() and self.food_rfid_device.connect_serial() and self.water_rfid_device.connect_serial()):
self.weight_device.close()
self.food_rfid_device.close()
self.water_rfid_device.close()
else:
log('error', "can't connect device with serials!")
print("Connect Device Error!")
self.food_work_thread = FoodWorkThread(
self.weight_device, self.food_rfid_device)
self.food_work_thread.update_uid.connect(self.food_rfid_update_uid)
self.food_work_thread.update_count.connect(self.food_rfid_update_count)
self.food_work_thread.update_val.connect(self.weight_update_value)
self.food_work_thread.update_date.connect(self.weight_update_date)
self.food_work_thread.update_table.connect(self.food_update_table)
self.food_work_thread.start()
# self.weight_listen_thread = Weight_Thread(self.weight_device)
# self.weight_listen_thread.update_date.connect(self.weight_update_date)
# self.weight_listen_thread.update_val.connect(self.weight_update_value)
# self.weight_listen_thread.start()
self.water_worker_thread = WaterWorkThread(
self.water_rfid_device
)
self.water_worker_thread.update_uid.connect(self.water_rfid_update_uid)
self.water_worker_thread.update_count.connect(self.water_rfid_update_count)
self.water_worker_thread.start()
def initApp():
global process, graph, best_model_filename
process = Process()
best_model_filename = getBestSavedModelToTest()
log('Best model : ' + str(best_model_filename) + ' picked for loading!')
# Load trained model
process.load(best_model_filename)
graph = tf.get_default_graph()
def indexATIS():
train_set, valid_set, dicts = load(
'atis.pkl') # load() from data_loader.py
w2idx, la2idx = dicts['words2idx'], dicts['labels2idx']
idx2w = {w2idx[k]: k for k in w2idx}
idx2la = {la2idx[k]: k for k in la2idx}
indexes = {
"idx2w": idx2w,
"idx2la": idx2la,
"w2idx": w2idx,
"la2idx": la2idx
}
with open('embeddings/word_indexes.json', 'w') as f:
json.dump(indexes, f)
log("Word Indexes saved at (embeddings/word_indexes.json)...")
train_x, _, train_label = train_set
valid_x, _, valid_label = valid_set
MAX_LEN = max(max([len(s) for s in train_x]),
max([len(s) for s in valid_x]))
# Add padding
train_x = pad_sequences(train_x,
maxlen=MAX_LEN,
padding='post',
value=w2idx["<UNK>"])
train_label = pad_sequences(train_label,
maxlen=MAX_LEN,
padding='post',
value=la2idx["O"])
valid_x = pad_sequences(valid_x,
maxlen=MAX_LEN,
padding='post',
value=w2idx["<UNK>"])
valid_label = pad_sequences(valid_label,
maxlen=MAX_LEN,
padding='post',
value=la2idx["O"])
train_set = (train_x, train_label) # packing only train_x and train_label
valid_set = (valid_x, valid_label)
return (train_set, valid_set, indexes)
def proc_exist(name):
try:
process = subprocess.Popen(
stdout=-1,
args='ps x|grep %s|grep -v grep|grep -v Z' % name,
shell=True)
output, _ = process.communicate()
process.poll()
if output:
logger.log('%s %s' % (output, _))
return set(i.split()[0]
for i in output.rstrip('\n').split('\n'))
return set()
except:
return set()
def load(self, filename):
custom_objects = {
'CRF': CRF,
'crf_loss': crf_loss,
'crf_viterbi_accuracy': crf_viterbi_accuracy
}
saved_model = load_model('trained_model/' + filename,
custom_objects=custom_objects)
log("Loaded model from disk", display=False)
highlight('white', 'Loaded model from disk')
self.model = saved_model
return saved_model
def food_main_listen(self):
food_pig_data = None
while True:
if ((self.food_rfid.update_uid != "None"
and self.food_rfid.update_uid != None
and len(self.food_rfid.update_uid) > 0)
and food_pig_data == None): # 刷入
# print(len(self.food_rfid.update_uid))
food_pig_data = PigData()
food_pig_data.in_time = dt.datetime.now().strftime("%H:%M:%S")
food_pig_data.tag_id = self.food_rfid.update_uid
food_pig_data._type = "food"
logstr = "[FOOD] - get inside - pig uid:" + \
str(food_pig_data.tag_id)
log('info', logstr)
print(logstr)
waittime = 0
while True:
if (waittime > 2):
food_pig_data = None
logstr = "[FOOD] wait for adding food TIMEOUT for 5 sec, delete this record"
log('info', logstr)
print(logstr)
break
if (self.food_device.device_val > 0.5): # 0.5kg
break
else:
waittime += 1
logstr = "[FOOD] wait for adding food to 500g, now just:" + \
str(self.food_device.device_val / 1000) + "g"
log('info', logstr)
print(logstr)
time.sleep(2.5)
change_pig = False
if (food_pig_data != None):
if (food_pig_data.tag_id != self.food_rfid.update_uid):
change_pig = True
if ((self.food_rfid.update_uid == "None" or change_pig)
and food_pig_data != None): # 刷出
food_pig_data.out_time = dt.datetime.now().strftime("%H:%M:%S")
food_pig_data.val = str(0.5 - self.food_device.device_val)
logstr = "[FOOD] - get outside - uid:" + \
str(food_pig_data.tag_id) + ", eat value: " + str(food_pig_data.val)
log('info', logstr)
print(logstr)
food_pig_data = None
time.sleep(0.001)
def parse(filename):
'''
Dataset schema:
sentence_idx | word | tag
'''
log("Parsing dataset ...")
data = readCSV(filename)
sentences = []
sentence_number = 0
sentence = []
words = {}
tags = {}
bar = progressbar.ProgressBar(maxval=len(data))
for line in bar(data):
sentence_idx = line[0].split(":")[1].strip() if line[0] != '' else ''
if str(sentence_number) != sentence_idx and sentence_idx != '':
if len(sentence) > 0:
sentences.append(sentence)
sentence = []
word, tag = line[1].lower(), line[-1]
# Generating a set of words & tags
if word not in words:
words[word] = True
if tag not in tags:
tags[tag] = True
sentence.append((word, tag))
words['<UNK>'] = True
train_set, valid_set, indexes = index(sentences, words, tags)
return (sentences, words, tags, train_set, valid_set, indexes)
def run(self):
self.rfid_thread = threading.Thread(target=self.run_rfid_thread)
self.weight_thread = threading.Thread(target=self.run_weight_thread)
self.rfid_thread.start()
self.weight_thread.start()
# rfid_onload = False
pig_data = None
while True:
self.update_uid.emit(self.food_rfid_device.update_uid)
self.update_count.emit(self.food_rfid_device.update_count)
# print(self.food_rfid_device.update_uid)
if ((self.food_rfid_device.update_uid != "None" and self.food_rfid_device.update_uid != None) and pig_data == None): # 刷入
pig_data = PigData()
pig_data.in_time = dt.datetime.now().strftime("%H:%M:%S")
pig_data.tag_id = self.food_rfid_device.update_uid
log('info', "get inside uid:" + str(pig_data.tag_id))
change_pig = False
if (pig_data != None):
if (pig_data.tag_id != self.food_rfid_device.update_uid):
change_pig = True
if ((self.food_rfid_device.update_uid == "None" or change_pig) and pig_data != None): # 刷出
pig_data.out_time = dt.datetime.now().strftime("%H:%M:%S")
pig_data.eat_val = str(self.weight_device.device_val)
self.update_table.emit(pig_data)
log('info', "get out uid:" + str(pig_data.tag_id) + ", eat value: " + str(pig_data.eat_val))
pig_data = None
self.update_date.emit(self.weight_device.device_date)
self.update_val.emit(self.weight_device.device_val)
# self.update_table.emit(self.weight_device.device_val)
time.sleep(1/23)
def validate(model_filename):
word_model = CustomEmbedding()
valid_set, indexes = word_model.valid_set, word_model.indexes
w2idx, la2idx = indexes['w2idx'], indexes['la2idx']
idx2w, idx2la = indexes['idx2w'], indexes['idx2la']
n_classes = len(idx2la)
n_vocab = len(idx2w)
valid_x, valid_label = valid_set
log("Processing word indexes... ")
words_val = [list(map(lambda x: idx2w[x], w)) for w in valid_x]
groundtruth_val = [list(map(lambda x: idx2la[x], y)) for y in valid_label]
log("Done processing word indexes!")
process = Process()
process.load(model_filename)
predword_val = process.validate(valid_set)
metrics = conlleval(predword_val, groundtruth_val, words_val, 'diff.txt')
log('Precision = {}, Recall = {}, F1 = {}'.format(metrics['precision'],
metrics['recall'],
metrics['f1']))
def water_main_listen():
water_pig_data = None
water_val_last = 0
while True:
if ((water_rfid.update_uid != "None"
and water_rfid.update_uid != None)
and water_pig_data == None): # 刷入
water_pig_data = PigData()
water_pig_data.in_time = dt.datetime.now().strftime("%H:%M:%S")
water_pig_data.tag_id = water_rfid.update_uid
water_pig_data._type = "water"
water_val_last = water_device.get_value()
logstr = "[WATER] - get inside - pig uid:" + \
str(water_pig_data.tag_id) + ", water val:" + str(water_val_last)
log('info', logstr)
print(logstr)
water_change_pig = False
if (water_pig_data != None):
if (water_pig_data.tag_id != water_rfid.update_uid):
water_change_pig = True
if ((water_rfid.update_uid == "None" or water_change_pig)
and water_pig_data != None): # 刷出
water_pig_data.out_time = dt.datetime.now().strftime(
"%H:%M:%S")
water_pig_data.val = water_device.get_value() - water_val_last
logstr = "[WATER] - get outsid - pig uid:" + \
str(water_pig_data.tag_id) + ", drink value: " + str(water_pig_data.val)
log('info', logstr)
print(logstr)
water_pig_data = None
time.sleep(0.001)
def water_main_listen(self):
water_pig_data = None
water_val_last = 0
while True:
try:
if ((self.water_rfid.update_uid != "None"
and self.water_rfid.update_uid != None)
and water_pig_data == None): # 刷入
water_pig_data = PigData()
water_pig_data.in_time = dt.datetime.now().strftime(
"%H:%M:%S")
water_pig_data.tag_id = self.water_rfid.update_uid
water_pig_data._type = "water"
water_val_last = self.water_device.get_value()
logstr = "[WATER] - get inside - pig uid:" + \
str(water_pig_data.tag_id) + ", water val:" + str(water_val_last)
log('info', logstr)
print(logstr)
water_change_pig = False
if (water_pig_data != None):
if (water_pig_data.tag_id != self.water_rfid.update_uid):
water_change_pig = True
if ((self.water_rfid.update_uid == "None" or water_change_pig)
and water_pig_data != None): # 刷出
water_pig_data.out_time = dt.datetime.now().strftime(
"%H:%M:%S")
water_pig_data.val = self.water_device.get_value(
) - water_val_last
logstr = "[WATER] - get outsid - pig uid:" + \
str(water_pig_data.tag_id) + ", drink value: " + str(water_pig_data.val)
log('info', logstr)
print(logstr)
water_pig_data = None
time.sleep(0.001)
except Exception as e:
print("WATER MAIN Thread Error:")
print(str(e))
log('error', "WATER MAIN Thread Error" + str(e))
def listen(self):
while True:
try:
if self.connect_serial() == False:
log('error', self.name + ": not connect to rfid device!")
print(self.name + ": not connect to rfid device!")
time.sleep(3)
continue
self.get_card()
log('debug', self.name + ": get card code:" + self.update_uid)
time.sleep(0.001)
except Exception as e:
print("RFID READ CARD ERROR:")
print(str(e))
log('error', self.name + " RFID Listen Error:" + str(e))
def index(sentences, words, tags):
log("Creating Indexes")
word2idx = {w: i for i, w in enumerate(words)}
la2idx = {tag: i
for i, tag in enumerate(tags)} # tag also called labels (la)
idx2word = {v: k for k, v in word2idx.items()}
idx2la = {v: k for k, v in la2idx.items()}
log("Spliting dataset into train and validation sets")
MAX_LEN = max([len(s) for s in sentences])
# w is of the form (word, label)
X = [[word2idx[w[0]] for w in s] for s in sentences]
X = pad_sequences(X, maxlen=MAX_LEN, padding='post', value=w2idx["<UNK>"])
y = [[la2idx[w[1]] for w in s] for s in sentences]
y = pad_sequences(y, maxlen=MAX_LEN, padding='post', value=la2idx["O"])
train_x, valid_x, train_label, valid_label = train_test_split(
X, y, test_size=0.33)
log("Done!")
indexes = {
"idx2w": idx2word,
"idx2la": idx2la,
"w2idx": word2idx,
"la2idx": la2idx
}
train_set = (train_x, train_label)
valid_set = (valid_x, valid_label)
return (train_set, valid_set, indexes)
def run_rfid_thread(self):
try:
self.rfid_device.listen()
except Exception as e:
log('error', "WATER RFID Listen Error:" + str(e))
pass
def save(self, filename):
self.model.save('trained_model/' + filename + '.h5')
log("Saved model to disk", display=False)
highlight('green', 'Saved model to disk')
from config import tagbox
from logs.logger import log
import requests
import json
import jwt
verbose = True
test = True
log('etl start')
# Extract data from Tagbox
tagbox_doc_endpoint = tagbox.document_endpoint
tagbox_tag_endpoint = tagbox.geotag_endpoint
#tagbox_params = {'doc_id': tagbox.test_doc_id}
#tagbox_params = tagbox.test_docs
tagbox_params = {}
tagbox_secret = tagbox.secret
jwt_algorithm = 'HS256'
jwt_headers = {'kid': tagbox.key}
tagbox_token = jwt.encode(tagbox_params, tagbox_secret, jwt_algorithm,
jwt_headers).decode()
tagbox_headers = {'Authorization': 'Bearer {}'.format(tagbox_token)}
if verbose:
print('Tagbox request:\n{}\n'.format('\n'.join([
tagbox_tag_endpoint,
str(tagbox_params),
def listen(self):
connect_count = 0
while True:
if (self.connect_serial() == False):
log('error', "FOOD_DEVICE: not connect device")
# print("FOOD_DEVICE: not connect device")
time.sleep(3)
continue
if not self.serial.in_waiting:
continue
try:
v = self.serial.readline().decode()
except serial.SerialException:
log('error', "weight device : SerialException!")
self.close()
try:
date_str = v.split('\r')[0]
date_str = dt.datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S")
self.device_date = date_str.strftime("%H:%M:%S")
log('debug', "weight device date : " + (self.device_date))
# print("date:" + self.device_date)
except Exception as err:
pass
try:
if ('kg' in v):
if ('-' in v):
v = v.split('-')[1]
kg = int(
v.split(' ')[4].replace('.', '').replace('kg\r\n', ''))
self.device_val = kg
# if (kg != 0):
# print(kg)
log('debug', "weight device val : " + str(self.device_val))
except Exception as e:
log('debug', "weight val error:" + str(e))
pass
# 防止數據堵塞
if self.device_val == 0:
connect_count += 1
if (connect_count > 60):
connect_count = 0
self.close()
time.sleep(0.1)
log('debug', "weight device : reconnect")
port=config.WEIGHT_PORT,
com=config.WEIGHT_COM)
water_rfid = RFID(ip=config.WATER_RFID_IP,
port=config.WATER_RFID_PORT,
com=config.WATER_RFID_COM,
name="WARTER_RFID")
water_device = Water(ip=config.WATER_IP,
port=config.WATER_PORT,
com=config.WATER_COM)
if (food_rfid.connect_serial() and food_device.connect_serial()
and water_rfid.connect_serial() and water_device.connect_serial()):
food_rfid.close()
food_device.close()
water_rfid.close()
water_device.close()
else:
log('error', "can't connect device with serials!")
print("Connect Device Error!")
food_multi = multiprocessing.Process(target=start_food_thread,
args=(food_rfid, food_device))
food_multi.start()
water_multi = multiprocessing.Process(target=start_water_thread,
args=(water_rfid, water_device))
water_multi.start()
while True:
time.sleep(1)
def run_weight_thread(self):
try:
self.weight_device.listen()
except Exception as e:
log('error', "Weight Listen Error:" + str(e))
def train(model=None, re_train=False):
word_model = CustomEmbedding(
re_train=re_train
) # To prevent creating embeddings if re_train is True
train_set, valid_set, indexes = word_model.train_set, word_model.valid_set, word_model.indexes
w2idx, la2idx = indexes['w2idx'], indexes['la2idx']
idx2w, idx2la = indexes['idx2w'], indexes['idx2la']
n_classes = len(idx2la)
n_vocab = len(idx2w)
train_x, train_label = train_set
valid_x, valid_label = valid_set
log("Processing word indexes... ")
words_val = [list(map(lambda x: idx2w[x], w)) for w in valid_x]
groundtruth_val = [list(map(lambda x: idx2la[x], y)) for y in valid_label]
log("Done processing word indexes!")
if re_train == False:
'''
DEFINE MODEL
'''
model = Sequential()
model.add(word_model.EmbeddingLayer())
model.add(Conv1D(128, 5, padding="same", activation='relu'))
model.add(Dropout(Config.DROPOUT))
model.add(
Bidirectional(
LSTM(units=Config.EMBEDDING_SIZE,
dropout=Config.DROPOUT,
recurrent_dropout=Config.DROPOUT,
kernel_initializer=he_normal(),
return_sequences=True)))
model.add(SeqSelfAttention(attention_activation='sigmoid'))
# model.add(GRU(units=Config.EMBEDDING_SIZE,
# dropout=Config.DROPOUT,
# recurrent_dropout=Config.DROPOUT,
# kernel_initializer=he_normal(),
# return_sequences=True))
model.add(TimeDistributed(Dense(n_classes, activation='softmax')))
model.add(CRF(n_classes, sparse_target=False, learn_mode='join'))
model.compile(Config.OPTIMIZER,
Config.LOSS,
metrics=[crf_viterbi_accuracy])
process = Process(model)
max_f1 = 0
try:
for i in range(Config.N_EPOCHS):
log("Epoch " + str(i + 1), display=False)
highlight('violet', 'Epoch ' + str(i + 1))
partition(80)
log("Training ")
process.train(train_set)
log("Validating ")
predword_val = process.validate(valid_set)
# Accuracy tests here using (predword_val, groundtruth_val, words_val) and save best model
metrics = conlleval(predword_val, groundtruth_val, words_val,
'diff.txt')
log('Precision = {}, Recall = {}, F1 = {}'.format(
metrics['precision'], metrics['recall'], metrics['f1']))
if metrics['f1'] > max_f1:
max_f1 = metrics['f1']
process.save('trained_model_' + str(Config.FILE_PATTERN) +
'_' + str(max_f1))
log("New model saved!", display=False)
highlight('white', 'Best validation F1 score : ' + str(max_f1))
log('Best validation F1 score : ' + str(max_f1), display=False)
log('Cleaning /trained_model folder...')
clean()
log('Removed all other saved models, kept the best model only!')
except KeyboardInterrupt: # If in case ctrl + c pressed, needs to clean up and exit
log("\nTraining interrupted with ctrl + c ...")
log('Cleaning /trained_model folder...')
clean()
log('Removed all other saved models, kept the best model only!')
sys.exit()
help="trains model and validates",
action="store_true")
parser.add_argument("--test",
help="tests model on an example input sentance",
action="store_true")
parser.add_argument("--validate",
help="Validate individual models for metrics",
action="store_true")
parser.add_argument("--retrain",
help="Re-train already saved models",
action="store_true")
args = parser.parse_args()
if args.train:
log('*** TRAINING ***' + '\n')
highlight(
'violet',
'Please open `logs/model.log` for all the logging information about the model'
)
train()
if args.test:
log('*** TESTING ***' + '\n')
test()
# For my own use (Can remove if you want)
if args.validate:
log('*** MODEL VALIDATION ***' + '\n')
def test(process=None, sentences=None, read_file=True):
start = time.time()
if read_file:
best_model_filename = getBestSavedModelToTest()
process = Process()
# Load trained model
process.load(best_model_filename)
f = open('tests/test_sentances.txt', 'r')
sentences = f.readlines()
f.close()
# Clean loaded sentances from file - removing '\n' from each sentance
sentences = process_sentances(sentences)
f = open('tests/slots_' + str(Config.FILE_PATTERN) + '.txt', 'w')
arr_slots = []
for sentence in sentences:
words, BIO = process.test(sentence) # Test on sentance
f.write(str(sentence) + "\n\n")
slots = {}
slot_type = None
value = ''
prev_slot_type = None
for idx, slot in enumerate(BIO):
if slot != 'O':
f.write(str(words[idx]) + " - " + str(slot) + "\n")
'''
Grouping the slots
san - B-toloc.city_name
francisco - I-toloc.city_name
------------------------------
Returns -> {'toloc.city_name': ['san francisco']}
'''
slot_type = slot.split("-")[1]
pos = slot.split("-")[0]
if pos == 'B':
if slot_type not in slots:
if prev_slot_type is not None:
slots[prev_slot_type].append(value.strip())
value = words[idx] + ' '
slots[slot_type] = []
prev_slot_type = slot_type
else:
slots[prev_slot_type].append(value.strip())
value = words[idx] + ' '
else: # pos == 'I'
value += words[idx] + ' '
slots[slot_type].append(value.strip())
log('Slots compiled into groups...')
f.write(partition(80) + "\n")
arr_slots.append(slots)
end = time.time()
f.close()
highlight('green', 'Output can be found in `slots.txt` file!')
response_time = end - start
if not read_file:
return (response_time, arr_slots[0]
) # As we're sending only one sentance in API URL
