def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Core = Model()
self.Helpers.logger.info("Class initialization complete.")
def __init__(self):
""" Initializes the class. """
super(RealsenseRead, self).__init__()
self.Helpers = Helpers("Realsense D415 Reader")
self.colorizer = rs.colorizer()
# OpenCV fonts
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.black = (0, 0, 0)
self.green = (0, 255, 0)
self.white = (255, 255, 255)
# Starts the Realsense module
self.Realsense = Realsense()
# Connects to the Realsense camera
self.profile = self.Realsense.connect()
# Starts the socket module
self.Socket = Socket("Realsense D415 Reader")
# Sets up the object detection model
self.Model = Model()
self.Helpers.logger.info(
"Realsense D415 Reader Class initialization complete.")
def do_http_classify(self):
""" Loads model and classifies test data """
# Load the model class
self.Model = Model()
# Classify the test data via the server
self.Model.test_http_classifier()
class Trainer():
""" Trainer Class
Trains the Natural Language Understanding Engine.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Train")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
self.Helpers.logger.info("Trainer class initialized.")
def setupData(self):
""" Prepares the data. """
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logger.info("NLU Training Data Ready")
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logger.info("NLU Trainer Entities Ready")
def trainModel(self):
""" Trains the model. """
while True:
self.Helpers.logger.info("Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
self.Helpers.logger.info("NLU Model Trained")
def model(self):
""" Initializes the model. """
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.tmodel = self.Model.buildDNN(self.x, self.y)
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Model = Model()
self.Helpers.logger.info(
"COVID19DN Tensorflow initialization complete.")
def do_classify(self):
""" Loads model and classifies test data """
# Load the model class
self.Model = Model()
# Load the model
self.do_load_model()
# Classify the test data
self.Model.test_classifier()
def do_server(self):
""" Loads the API server """
# Load the model class
self.Model = Model()
# Load the model
self.do_load_model()
# Load the server class
self.Server = Server(self.Model)
# Start the server
self.Server.start()
def do_train(self):
""" Creates & trains the model. """
# Load the model class
self.Model = Model()
# Create the model
self.Model.do_model()
# Train the model
self.Model.do_train()
# Validate the model
self.Model.do_evaluate()
def __init__(self):
""" Initializes the Training class. """
self.Helpers = Helpers()
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def __init__(self):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Train")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
self.Helpers.logger.info("Trainer class initialized.")
def do_model(self):
""" Creates & trains the model.
Replicates the networked and data splits outlined in the Acute Leukemia Classification
Using Convolution Neural Network In Clinical Decision Support System paper
using Tensorflow 2.0.
https://airccj.org/CSCP/vol7/csit77505.pdf
"""
self.Model = Model(self.optimizer, self.do_augmentation)
self.Model.build_network(self.Data.X_train, self.Data.X_test,
self.Data.y_train, self.Data.y_test)
self.Model.compile_and_train()
self.Model.save_model_as_json()
self.Model.save_weights()
def initNLU(self):
""" Initializes a Chatbot sesiion.
Initiates the NLU setting up the data, NLU / entities models
and required modules such as context and extensions.
"""
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
class ALLoneAPI():
""" ALLoneAPI RPI4 CNN Core Class
Core class for the oneAPI RPI4 Acute Lymphoblastic Leukemia Classifier.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Core = Model()
self.Helpers.logger.info("ALLoneAPI RPI4 CNN initialization complete.")
def do_load_model(self):
""" Loads the model """
self.Core.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_load_model()
self.Core.test_classifier()
def do_server(self):
""" Loads the API server """
self.do_load_model()
self.Server = Server(self.Core)
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
self.Core.test_http_classifier()
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
# Initiates the iotJumpWay connection class
self.iotJumpWayDevice = iotJumpWay({
"host": self.Helpers.confs["iotJumpWay"]["host"],
"port": self.Helpers.confs["iotJumpWay"]["port"],
"lid": self.Helpers.confs["iotJumpWay"]["loc"],
"zid": self.Helpers.confs["iotJumpWay"]["zne"],
"did": self.Helpers.confs["iotJumpWay"]["id"],
"dn": self.Helpers.confs["iotJumpWay"]["name"],
"un": self.Helpers.confs["iotJumpWay"]["mqtt"]["username"],
"pw": self.Helpers.confs["iotJumpWay"]["mqtt"]["password"]
})
self.iotJumpWayDevice.connect()
self.Model = Model(self.iotJumpWayDevice)
self.Helpers.logger.info(
"COVID19DN Tensorflow initialization complete.")
class AllDS2020():
""" AllDS2020 CNN For Raspberry Pi 4 Class
Core AllDS2020 CNN For Raspberry Pi 4 Tensorflow 2.0 class.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Core = Model()
self.Helpers.logger.info("AllDS2020 CNN initialization complete.")
def do_load_model(self):
""" Loads the model """
self.Core.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_load_model()
self.Core.test_classifier()
def do_server(self):
""" Loads the API server """
self.do_load_model()
self.Server = Server(self.Core)
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
self.Core.test_http_classifier()
def do_create_model(self):
""" Loads the model """
self.Model = Model(self.optimizer, self.do_augmentation)
class Chatbot():
""" ALL Detection System 2019 Chatbot Class
The ALL Detection System 2019 Chatbot.
"""
def __init__(self):
""" Initializes the Chatbot class. """
self.isTraining = False
self.ner = None
self.Helpers = Helpers()
self.user = {}
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "NLU/")
self.ChatLogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Chat/")
def initiateSession(self):
""" Initializes a Chatbot sesiion.
Initiates empty guest user session, GeniSys will ask the user
verify their GeniSys user by speaking or typing if it does
not know who it is speaking to.
"""
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def initNLU(self):
""" Initializes a Chatbot sesiion.
Initiates the NLU setting up the data, NLU / entities models
and required modules such as context and extensions.
"""
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
def commandsCallback(self, topic, payload):
""" iotJumpWay callback function.
The callback function that is triggerend in the event of a
command communication from the iotJumpWay.
"""
self.Helpers.logMessage(
self.LogFile, "iotJumpWay", "INFO",
"Recieved iotJumpWay Command Data : " + str(payload))
commandData = json.loads(payload.decode("utf-8"))
def restoreData(self):
""" Restores the training data.
Sets the local trained data using data retrieved above
"""
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def loadEntityController(self):
""" Initiates the entity extractor class """
self.entityController = Entities()
def restoreNER(self):
""" Loads entity controller and restores the NER model """
self.loadEntityController()
self.ner = self.entityController.restoreNER()
def restoreNLU(self):
""" Restores the NLU model """
self.tmodel = self.Model.buildDNN(self.x, self.y)
def setThresholds(self):
""" Sets thresholds
Sets the threshold for the NLU engine, this can be changed
using arguments to commandline programs or paramters for
API calls.
"""
self.threshold = self.Helpers.confs["NLU"]["Threshold"]
self.entityThrshld = self.Helpers.confs["NLU"]["Mitie"]["Threshold"]
def communicate(self, sentence):
""" Responds to the user
First checks to ensure that the program is not training,
then parses any entities that may be in the intent, then
checks context and extensions before providing a response.
"""
if self.isTraining == False:
parsed, fallback, entityHolder, parsedSentence = self.entityController.parseEntities(
sentence, self.ner, self.trainingData)
classification = self.Model.predict(self.tmodel, parsedSentence,
self.trainedWords,
self.trainedClasses)
if len(classification) > 0:
clearEntities = False
theIntent = self.trainingData["intents"][self.intentMap[
classification[0][0]]]
if len(entityHolder) and not len(theIntent["entities"]):
clearEntities = True
if (self.Context.checkSessionContext(self.user[self.userID],
theIntent)):
if self.Context.checkClearContext(theIntent, 0):
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entityHolder = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and fallback in theIntent and len(
theIntent["fallbacks"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses = None, []
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
if (type(response) == tuple):
response = response[0]
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
contextCurrent = self.Context.getCurrentContext(
self.user[self.userID])
return {
"Response":
"FAILED",
"ResponseData": [{
"Received":
sentence,
"Intent":
"UNKNOWN",
"Confidence":
"NA",
"Responses": [],
"Response":
random.choice(
self.Helpers.confs["NLU"]["defaultResponses"]),
"Context": [{
"In": "NA",
"Out": "NA",
"Current": contextCurrent
}],
"Extension":
"NA",
"Entities":
entityHolder
}]
}
else:
return {
"Response":
"FAILED",
"ResponseData": [{
"Status": "Training",
"Message": "NLU Engine is currently training"
}]
}
class AllDS2020():
""" AllDS2020 Wrapper Class
Core wrapper class for the Tensorflow 2.0 AllDS2020 classifier.
"""
def __init__(self):
self.Helpers = Helpers("Core")
self.optimizer = "Adam"
self.mode = "Local"
self.do_augmentation = True
def do_data(self):
""" Creates/sorts dataset. """
self.Data = Data(self.optimizer, self.do_augmentation)
self.Data.data_and_labels_sort()
if self.do_augmentation == False:
self.Data.data_and_labels_prepare()
else:
self.Data.data_and_labels_augmentation_prepare()
self.Data.shuffle()
self.Data.get_split()
def do_model(self):
""" Creates & trains the model.
Replicates the networked and data splits outlined in the Acute Leukemia Classification
Using Convolution Neural Network In Clinical Decision Support System paper
using Tensorflow 2.0.
https://airccj.org/CSCP/vol7/csit77505.pdf
"""
self.Model = Model(self.optimizer, self.do_augmentation)
self.Model.build_network(self.Data.X_train, self.Data.X_test,
self.Data.y_train, self.Data.y_test)
self.Model.compile_and_train()
self.Model.save_model_as_json()
self.Model.save_weights()
def do_evaluate(self):
""" Predictions & Evaluation """
self.Model.predictions()
self.Model.evaluate_model()
def do_metrics(self):
""" Predictions & Evaluation """
self.Model.visualize_metrics()
self.Model.confusion_matrix()
self.Model.figures_of_merit()
def do_create_model(self):
""" Loads the model """
self.Model = Model(self.optimizer, self.do_augmentation)
def do_load_model(self):
""" Loads the model """
self.Model.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_create_model()
self.do_load_model()
self.Model.test_classifier()
def do_http_classify(self):
""" Loads model and classifies test data """
self.do_create_model()
self.Model.test_http_classifier()
class RealsenseRead(Thread):
""" Realsense D415 Reader Class
Realsense D415 reader functions for EMAR Mini Emergency Assistance Robot.
"""
def __init__(self):
""" Initializes the class. """
super(RealsenseRead, self).__init__()
self.Helpers = Helpers("Realsense D415 Reader")
self.colorizer = rs.colorizer()
# OpenCV fonts
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.black = (0, 0, 0)
self.green = (0, 255, 0)
# Starts the Realsense module
self.Realsense = Realsense()
# Connects to the Realsense camera
self.profile = self.Realsense.connect()
# Starts the socket module
self.Socket = Socket("Realsense D415 Reader")
# Sets up the object detection model
self.Model = Model()
self.Helpers.logger.info("Realsense D415 Reader Class initialization complete.")
def run(self):
""" Runs the module. """
t1 = 0
t2 = 0
fc = 0
dc = 0
fps = ""
dfps = ""
# Starts the socket server
soc = self.Socket.connect(self.Helpers.confs["EMAR"]["ip"],
self.Helpers.confs["Realsense"]["socket"]["port"])
try:
while True:
t1 = time.perf_counter()
# Wait for a coherent pair of frames: depth and color
frames = self.Realsense.pipeline.wait_for_frames()
depth_frame = frames.get_depth_frame()
color_frame = frames.get_color_frame()
#ir1_frame = frames.get_infrared_frame(1)
#ir2_frame = frames.get_infrared_frame(2)
if not depth_frame or not color_frame:
#if not not ir1_frame or not ir2_frame:
self.Helpers.logger.info("Realsense D415 streams not ready, continuing.")
continue
# Convert images to numpy arrays
depth_image = np.asanyarray(depth_frame.get_data())
color_image = np.asanyarray(color_frame.get_data())
#ir1_image = np.asanyarray(self.colorizer.colorize(ir1_frame).get_data())
#ir2_image = np.asanyarray(self.colorizer.colorize(ir2_frame).get_data())
colorized_depth = np.asanyarray(self.colorizer.colorize(depth_frame).get_data())
width, height = self.Model.getDims(color_image)
self.Model.setBlob(color_image)
detections = self.Model.forwardPass()
# Writes header to frame
cv2.putText(color_image, "EMAR Mini Color Stream", (30,50), self.font,
0.7, self.black, 2, cv2.LINE_AA)
# Writes date to frame
cv2.putText(color_image, str(datetime.now()), (30,80), self.font,
0.5, self.black, 2, cv2.LINE_AA)
# Writes header to frame
cv2.putText(colorized_depth, "EMAR Mini Depth Stream", (30,50), self.font,
0.7, self.black, 2, cv2.LINE_AA)
# Writes date to frame
cv2.putText(colorized_depth, str(datetime.now()), (30,80), self.font,
0.5, self.black, 2, cv2.LINE_AA)
for i in range(100):
bi = i * 7
if i == 0:
dc += 1
# Object location
x1 = max(0, int(detections[bi + 3] * height))
y1 = max(0, int(detections[bi + 4] * width))
x2 = min(height, int(detections[bi + 5] * height))
y2 = min(width, int(detections[bi + 6] * width))
overlay = detections[bi:bi + 7]
bi = 0
class_id = overlay[bi + 1]
percentage = int(overlay[bi + 2] * 100)
if (percentage <= self.Helpers.confs["MobileNetSSD"]["threshold"]):
continue
# Calculates box coordinates
box_left = int(overlay[bi + 3] * width)
box_top = int(overlay[bi + 4] * height)
box_right = int(overlay[bi + 5] * width)
box_bottom = int(overlay[bi + 6] * height)
# Gets the meters from the depth frame
meters = depth_frame.as_depth_frame().get_distance(box_left+int((box_right-box_left)/2),
box_top+int((box_bottom-box_top)/2))
if meters != 0.00:
label_text = self.Helpers.confs["MobileNetSSD"]["classes"][int(class_id)] + " (" + str(percentage) + "%)"+ " {:.2f}".format(meters) + "m"
else:
label_text = self.Helpers.confs["MobileNetSSD"]["classes"][int(class_id)] + " (" + str(percentage) + "%)"
cv2.rectangle(color_image, (box_left, box_top), (box_right, box_bottom),
self.green, 1)
# Positions and writes the label
label_size = cv2.getTextSize(label_text, self.font, 0.5, 1)[0]
label_left = box_left
label_top = box_top - label_size[1]
if (label_top < 1):
label_top = 1
label_right = label_left + label_size[0]
label_bottom = label_top + label_size[1]
cv2.putText(color_image, label_text, (label_left, label_bottom), self.font, 0.5,
self.green, 1)
# Writes FPS and Detection Frames Per Second
cv2.putText(color_image, fps, (width-170,20), self.font, 0.5,
self.black, 1, cv2.LINE_AA)
cv2.putText(color_image, dfps, (width-170,35), self.font, 0.5,
self.black, 1, cv2.LINE_AA)
# Combine the color_image and colorized_depth frames together:
frame = np.hstack((color_image, colorized_depth))
# Combine the ir1_image and ir2_image frames together:
#frame = np.hstack((ir1_image, ir2_image))
# Streams the modified frame to the socket server
encoded, buffer = cv2.imencode('.jpg', frame)
soc.send(base64.b64encode(buffer))
# FPS calculation
fc += 1
if fc >= 15:
fps = "Stream: {:.1f} FPS".format(t1/15)
dfps = "Detection: {:.1f} FPS".format(dc/t2)
fc = 0
dc = 0
t1 = 0
t2 = 0
t2 = time.perf_counter()
elapsedTime = t2-t1
t1 += 1/elapsedTime
t2 += elapsedTime
finally:
# Stop streaming
self.Realsense.pipeline.stop()
class Trainer():
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
def __init__(self):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
if self._confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self._confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
class COVID19DN():
""" COVID19DN Class
Core COVID-19 Tensorflow DenseNet Classifier wrapper class using Tensroflow 2.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Helpers.logger.info(
"COVID-19 Tensorflow DenseNet Classifier initialization complete.")
def life(self):
""" Sends vital statistics to HIAS """
cpu = psutil.cpu_percent()
mem = psutil.virtual_memory()[2]
hdd = psutil.disk_usage('/').percent
tmp = psutil.sensors_temperatures()['cpu-thermal'][0].current
r = requests.get('http://ipinfo.io/json?token=' +
self.Helpers.confs["iotJumpWay"]["key"])
data = r.json()
location = data["loc"].split(',')
self.Helpers.logger.info("COVID19DN Life (TEMPERATURE): " + str(tmp) +
"\u00b0")
self.Helpers.logger.info("COVID19DN Life (CPU): " + str(cpu) + "%")
self.Helpers.logger.info("COVID19DN Life (Memory): " + str(mem) + "%")
self.Helpers.logger.info("COVID19DN Life (HDD): " + str(hdd) + "%")
self.Helpers.logger.info("COVID19DN Life (LAT): " + str(location[0]))
self.Helpers.logger.info("COVID19DN Life (LNG): " + str(location[1]))
# Send iotJumpWay notification
self.iotJumpWayDevice.devicePub(
"Life", {
"CPU": cpu,
"Memory": mem,
"Diskspace": hdd,
"Temperature": tmp,
"Latitude": location[0],
"Longitude": location[1]
})
threading.Timer(60.0, self.life).start()
def iotjumpway_client(self):
""" Starts iotJumpWay Client. """
# Initiates the iotJumpWay connection class
self.iotJumpWayDevice = iotJumpWay({
"host":
self.Helpers.confs["iotJumpWay"]["host"],
"port":
self.Helpers.confs["iotJumpWay"]["port"],
"lid":
self.Helpers.confs["iotJumpWay"]["loc"],
"zid":
self.Helpers.confs["iotJumpWay"]["zne"],
"did":
self.Helpers.confs["iotJumpWay"]["id"],
"dn":
self.Helpers.confs["iotJumpWay"]["name"],
"un":
self.Helpers.confs["iotJumpWay"]["mqtt"]["username"],
"pw":
self.Helpers.confs["iotJumpWay"]["mqtt"]["password"]
})
self.iotJumpWayDevice.connect()
def threading(self):
""" Creates required module threads. """
# Life thread
Thread(target=self.life, args=()).start()
threading.Timer(60.0, self.life).start()
def do_train(self):
""" Creates & trains the model. """
# Load the model class
self.Model = Model()
# Create the model
self.Model.do_model()
# Train the model
self.Model.do_train()
# Validate the model
self.Model.do_evaluate()
def do_load_model(self):
""" Loads the model """
# Load the model and weights
self.Model.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
# Load the model class
self.Model = Model()
# Load the model
self.do_load_model()
# Classify the test data
self.Model.test_classifier()
def do_server(self):
""" Loads the API server """
# Load the model class
self.Model = Model()
# Load the model
self.do_load_model()
# Load the server class
self.Server = Server(self.Model)
# Start the server
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
# Load the model class
self.Model = Model()
# Classify the test data via the server
self.Model.test_http_classifier()
class RealsenseRead(Thread):
""" Realsense D415 Reader Class
Realsense D415 reader functions for EMAR Mini Emergency Assistance Robot.
"""
def __init__(self):
""" Initializes the class. """
super(RealsenseRead, self).__init__()
self.Helpers = Helpers("Realsense D415 Reader")
self.colorizer = rs.colorizer()
# OpenCV fonts
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.black = (0, 0, 0)
self.green = (0, 255, 0)
self.white = (255, 255, 255)
# Starts the Realsense module
self.Realsense = Realsense()
# Connects to the Realsense camera
self.profile = self.Realsense.connect()
# Starts the socket module
self.Socket = Socket("Realsense D415 Reader")
# Sets up the object detection model
self.Model = Model()
self.Helpers.logger.info(
"Realsense D415 Reader Class initialization complete.")
def run(self):
""" Runs the module. """
time1 = 0
time2 = 0
fc = 0
dc = 0
fps = ""
dfps = ""
# Starts the socket server
soc = self.Socket.connect(
self.Helpers.confs["EMAR"]["ip"],
self.Helpers.confs["Realsense"]["socket"]["port"])
try:
while True:
t1 = time.perf_counter()
# Wait for depth and color frames
frames = self.Realsense.pipeline.wait_for_frames()
# Get color frames
depth_frame = frames.get_depth_frame()
color_frame = frames.get_color_frame()
# Get infrared frames
#ir1_frame = frames.get_infrared_frame(1)
#ir2_frame = frames.get_infrared_frame(2)
if not depth_frame or not color_frame:
#if not not ir1_frame or not ir2_frame:
self.Helpers.logger.info(
"Realsense D415 streams not ready, continuing.")
continue
# Convert images to numpy arrays
depth_image = np.asanyarray(depth_frame.get_data())
color_image = np.asanyarray(color_frame.get_data())
#ir1_image = np.asanyarray(self.colorizer.colorize(ir1_frame).get_data())
#ir2_image = np.asanyarray(self.colorizer.colorize(ir2_frame).get_data())
colorized_depth = np.asanyarray(
self.colorizer.colorize(depth_frame).get_data())
# Sets the blob and gets the detections from forward pass
self.Model.setBlob(color_image)
detections = self.Model.forwardPass()
# Gets width, height and crop value for frame
width, height = self.Model.getDims(color_image)
# Writes header to frame
cv2.putText(color_image, "EMAR Mini Color Stream", (30, 50),
self.font, 0.7, self.black, 2, cv2.LINE_AA)
# Writes date to frame
cv2.putText(color_image, str(datetime.now()), (30, 80),
self.font, 0.5, self.black, 2, cv2.LINE_AA)
# Writes header to frame
cv2.putText(colorized_depth, "EMAR Mini Depth Stream",
(30, 50), self.font, 0.7, self.white, 2,
cv2.LINE_AA)
# Writes date to frame
cv2.putText(colorized_depth, str(datetime.now()), (30, 80),
self.font, 0.5, self.white, 2, cv2.LINE_AA)
for i in range(detections.shape[2]):
# Detection FPS
if i == 0:
dc += 1
# Gets and checks confidence of detection
confidence = detections[0, 0, i, 2]
if (confidence <=
self.Helpers.confs["MobileNetSSD"]["threshold"]):
continue
# Gets class of detection
class_id = int(detections[0, 0, i, 1])
# Gets bounding box of detection
box_blx = int(detections[0, 0, i, 3] * width)
box_bly = int(detections[0, 0, i, 4] * height)
box_trx = int(detections[0, 0, i, 5] * width)
box_try = int(detections[0, 0, i, 6] * height)
# Gets the meters from the depth frame
meters = depth_frame.as_depth_frame().get_distance(
box_blx + int((box_trx - box_blx) / 2), box_bly + int(
(box_try - box_bly) / 2))
# Prepares label text
if meters != 0.00:
label_text = self.Helpers.confs["MobileNetSSD"][
"classes"][int(class_id)] + " (" + str(
confidence) + "%)" + " {:.2f}".format(
meters) + "m"
else:
label_text = self.Helpers.confs["MobileNetSSD"][
"classes"][int(class_id)] + " (" + str(
confidence) + "%)"
cv2.rectangle(color_image, (box_blx, box_bly),
(box_trx, box_try), self.green, 1)
# Positions and writes the label
label_size = cv2.getTextSize(label_text, self.font, 0.5,
1)[0]
label_left = box_blx
label_top = box_bly - label_size[1]
if (label_top < 1):
label_top = 1
label_bottom = label_top + label_size[1]
cv2.putText(color_image, label_text,
(label_left, label_bottom), self.font, 0.5,
self.green, 1)
# Writes FPS and Detection FPS
cv2.putText(color_image, fps, (width - 170, 50), self.font,
0.5, self.black, 1, cv2.LINE_AA)
cv2.putText(color_image, dfps, (width - 170, 65), self.font,
0.5, self.black, 1, cv2.LINE_AA)
# Combine the color_image and colorized_depth frames together:
frame = np.hstack((color_image, colorized_depth))
# Combine the ir1_image and ir2_image frames together:
#frame = np.hstack((ir1_image, ir2_image))
# Streams the modified frame to the socket server
encoded, buffer = cv2.imencode('.jpg', frame)
soc.send(base64.b64encode(buffer))
# FPS calculation
fc += 1
if fc >= 15:
fps = "Stream: {:.1f} FPS".format(time1 / 15)
dfps = "Detection: {:.1f} FPS".format(dc / time2)
fc = 0
dc = 0
time1 = 0
time2 = 0
t2 = time.perf_counter()
elapsedTime = t2 - t1
time1 += 1 / elapsedTime
time2 += elapsedTime
finally:
# Stop streaming
self.Realsense.pipeline.stop()
class Engine():
""" Engine Class
Core functions for the NLU Engine.
"""
def __init__(self, isAudio):
""" Initializes the class. """
self.Helpers = Helpers("Engine")
self.ner = None
self.user = {}
#self.bluetoothCon()
self.data()
self.entities()
#self.iotJumpWayCon()
self.model()
self.session()
self.thresholds()
if isAudio:
self.speech()
self.Helpers.logger.info("Engine class initialized.")
def bluetoothCon(self):
""" Initializes the Bluetooth connection. """
self.Bluetooth = BluetoothConnect()
self.Bluetooth.connect()
def data(self):
""" Initializes the data. """
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def doExtension(self, extension, entities, exEntities, extensionResponses):
""" Executes an extension. """
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass, classParts[2])(extensionResponses,
theEntities)
return response
def entities(self):
""" Initializes the entities. """
self.entityController = Entities()
self.ner = self.entityController.restoreNER()
def entitiesCheck(self, entityHolder, theIntent, clearEntities):
""" Checks entities. """
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entities = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
return response, entities, extension, extensionResponses, exEntities
def fallbackCheck(self, fallback, theIntent, entityHolder):
""" Checks if fallback. """
if fallback and fallback in theIntent and len(theIntent["fallbacks"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
extension, extensionResponses, exEntities = None, [], None
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
return response, entities, extension, extensionResponses, exEntities
def model(self):
""" Initializes the model. """
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.tmodel = self.Model.buildDNN(self.x, self.y)
def session(self):
""" Initializes a NLU sesiion.
Initiates empty guest user session, GeniSys will ask the user
verify their GeniSys user by speaking or typing if it does
not know who it is speaking to.
"""
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def respond(self, status, sentence, intent, confidence, response, cIn,
cOut, cCurrent, extension, entities):
""" Forms the response. """
return {
"Response":
status,
"ResponseData": [{
"Received":
sentence,
"Intent":
intent,
"Confidence":
confidence,
"Response":
response,
"Context": [{
"In": cIn,
"Out": cOut,
"Current": cCurrent
}],
"Extension":
extension,
"Entities":
entities
}]
}
def speech(self):
""" Initializes the TTS feature. """
self.TTS = TTS()
def thresholds(self):
""" Sets thresholds
Sets the threshold for the NLU engine, this can be changed
using arguments to commandline programs or paramters for
API calls.
"""
self.threshold = self.Helpers.confs["NLU"]["Threshold"]
self.entityThrshld = self.Helpers.confs["NLU"]["Mitie"]["Threshold"]
class COVID19DN():
""" COVID19DN Class
Core COVID-19 Tensorflow DenseNet Classifier wrapper class using Tensroflow 2.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Model = Model()
self.Helpers.logger.info(
"COVID19DN Tensorflow initialization complete.")
def do_data(self):
""" Sorts the training data. """
self.Data = Data()
self.Data.process_data(
self.Data.paths_n_labels())
def do_train(self):
""" Creates & trains the model. """
self.Model.do_model(self.Data)
self.Model.do_train()
self.Model.do_evaluate()
def do_load_model(self):
""" Loads the model """
self.Model.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_load_model()
self.Model.test_classifier()
def do_server(self):
""" Loads the API server """
self.do_load_model()
self.Server = Server(self.Model)
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
self.Model.test_http_classifier()
class ALLoneAPI():
""" ALLoneAPI CNN
Core class for the OneAPI Acute Lymphoblastic Leukemia Classifier CNN.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Core = Model()
self.Helpers.logger.info("Class initialization complete.")
def do_train(self):
""" Creates & trains the model. """
self.Core.do_data()
self.Core.do_network()
self.Core.do_train()
self.Core.do_evaluate()
def do_load_model(self):
""" Loads the model """
self.Core.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_load_model()
self.Core.test_classifier()
def do_server(self):
""" Loads the API server """
self.do_load_model()
self.Server = Server(self.Core)
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
self.Core.test_http_classifier()
class Trainer():
""" ALL Detection System 2019 Chatbot Training Class
Trains the ALL Detection System 2019 Chatbot.
"""
def __init__(self):
""" Initializes the Training class. """
self.Helpers = Helpers()
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
""" Prepares the data. """
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
""" Trains the model. """
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
class AllDS2020():
""" AllDS2020 CNN Class
Core AllDS2020 CNN Tensorflow 2.0 class.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Core")
self.Core = Model()
self.Helpers.logger.info("AllDS2020 CNN initialization complete.")
def do_train(self):
""" Creates & trains the model. """
self.Core.do_data()
self.Core.do_network()
self.Core.do_train()
self.Core.do_evaluate()
def do_load_model(self):
""" Loads the model """
self.Core.load_model_and_weights()
def do_classify(self):
""" Loads model and classifies test data """
self.do_load_model()
self.Core.test_classifier()
def do_server(self):
""" Loads the API server """
self.do_load_model()
self.Server = Server(self.Core)
self.Server.start()
def do_http_classify(self):
""" Loads model and classifies test data """
self.Core.test_http_classifier()
# Naipsas (Btc Sources)
#
# Upster Project - GitHub
#
# import pygame as pg
import os
import sys
# from Module import function
from Classes.View import View
from Classes.Model import Model
if __name__ == "__main__":
try:
GuiManager = View()
ModelManager = Model()
# Creamos la interfaz gráfica
view = GuiManager.createMainWindow()
# Adjuntamos la lógica
# Ponemos la interfaz en funcionamiento
GuiManager.startWindowWorking()
except Exception as e:
print(str(e))