def __init__(self, initialization_tuple):
child_pipe, config, logging_queue = initialization_tuple
self.child_pipe = child_pipe
self._config = config
self._master_host = config["DEFAULT"]["master_domain"]
self._master_port = config["DEFAULT"]["master_port"]
self._log_dir = config["LOGGING"]["log_dir"]
self._root_dir = config["DEFAULT"]["root_dir"]
self._node_name = config["DEFAULT"].get("name", "node")
self._script_dir = config["DEFAULT"].get("scripts_dir",
self._root_dir + "/scripts")
self._private_key_password = config["DEFAULT"]["private_key_password"]
# setup logging
qh = logging.handlers.QueueHandler(logging_queue)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(qh)
self.logger = logging.getLogger("NodeClientProcessLogger")
self.logger.setLevel(logging.DEBUG)
self.logger.info(
"NodeListenerProcess Inialized. Creating Connection To SQL DB")
self._sql_manager = SQLiteManager(config, self.logger)
self.logger.info("Connection Complete")
def handle_master_requests(self, command):
# TODO: This is called if there is an error made by a request sent by master process. If master process
# TODO: makes calls using the pipes, it would be able to wait at the calling code point for the response
# TODO: this may be more ideal ??
if command['command'] == "ERROR":
self._logger.error("Error Command Received")
self._logger.error(command)
return None
if command['command'] == "EXEC" and command['params'] == "SCAN.SCRIPTS":
sqlite_manager = SQLiteManager(self._config, self._logger)
self.catalogue_local_scripts(sqlite_manager)
sqlite_manager.closeEverything()
old_from = command['from']
command['from'] = command['to']
command['to'] = old_from
command['param'] = "SUCCESS"
command['rawdata'] = ""
return command
if command['command'] == "EXEC" and command['params'] == "SCRIPTS.EXECUTE":
sqlite_manager = SQLiteManager(self._config, self._logger)
response = taskrunner.execute_script_on_node(sqlite_manager, command, self._logger)
sqlite_manager.closeEverything()
return response
if command['command'] == "GET" and command['params'] == "PING":
return taskrunner.get_ping_info(command, self._config)
def GETAllNodes():
self.logger.info("Fetching All Nodes")
sql_manager = SQLiteManager(self._config, self.logger)
all_nodes = sql_manager.getAllNodes()
all_nodes_as_dictionaries = list()
for node in all_nodes:
dict_node = node.toDictionary()
dict_node.pop("ip", None)
all_nodes_as_dictionaries.append(dict_node)
sql_manager.closeEverything()
return jsonify(all_nodes_as_dictionaries)
def GETAllScripts():
self.logger.info("Fetching All Scripts")
sql_manager = SQLiteManager(self._config, self.logger)
all_scripts = sql_manager.getAllScripts()
all_scripts_as_dictionaries = list()
for script in all_scripts:
dict_script = script.toDictionary()
dict_script.pop("file_path", None)
all_scripts_as_dictionaries.append(dict_script)
sql_manager.closeEverything()
return jsonify(all_scripts_as_dictionaries)
def POSTMigrateScriptToNode(script_guid):
node_guid = request.json["node_guid"]
if node_guid is None:
abort(400, "A Node Guid Is Required For Migration")
try:
uuid.UUID(script_guid, version=4)
except:
abort(400, "The Passed In Script Guid Is Invalid")
try:
uuid.UUID(node_guid, version=4)
except:
abort(400, "The Passed In Node Guid Is Invalid")
self.logger.info("Migrating Script Of Guid: " + script_guid + " To Node Of Guid: " + node_guid)
sql_manager = SQLiteManager(self._config, self.logger)
all_scripts = sql_manager.getAllScripts()
for script in all_scripts:
if script.guid == uuid.UUID(script_guid):
# found our script
action = dict()
action['command'] = "MIG"
action['from'] = "HTTP"
action['to'] = "NODE"
action['params'] = (str(node_guid), script.toDictionary())
# rag the raw file content and put into binary string
file_path = script.file_path + os.sep + script.file_name
fp = open(file_path, 'rb')
all_file_contents = fp.read()
fp.close()
action['rawdata'] = base64.b64encode(all_file_contents).decode('utf-8')
self._pipe_lock.acquire()
self.child_pipe.send(action)
answer = self.child_pipe.recv()
self._pipe_lock.release()
sql_manager.closeEverything()
if answer['command'] == "ERROR":
return handle_internal_error(answer)
else:
response = dict()
response["migrationStatus"] = answer["params"]
response["fileName"] = answer['rawdata']["file_name"]
response["destinationFilePath"] = answer['rawdata']["file_path"]
response["scriptGuid"] = answer['rawdata']["guid"]
return jsonify(response)
sql_manager.closeEverything()
abort(404)
def __init__(self, initialization_tuple):
child_pipe, config, logging_queue = initialization_tuple
self.child_pipe = child_pipe
self._config = config
self._port = config["NODELISTENER"]["port"]
self._bind_ip = config["NODELISTENER"]["bind_ip"]
self._log_dir = config["LOGGING"]["log_dir"]
self.private_key_password = config["DEFAULT"]["private_key_password"]
qh = logging.handlers.QueueHandler(logging_queue)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(qh)
self.logger = logging.getLogger("NodeListenerProcessLogger")
self.logger.setLevel(logging.DEBUG)
self.logger.info("NodeListenerProcess Inialized. Creating Connection To SQL DB")
self._sql_manager = SQLiteManager(config, self.logger)
self.logger.info("Connection Complete")
def main(self):
self._logger.info("Service Is Initializing...")
# setup database
sqlite_manager = SQLiteManager(self._config, self._logger)
# catalogue all the scripts in the system
self._logger.info("Catalogueing Engines On The System")
sm.catalogue_local_engines(sqlite_manager, self._logger)
self._logger.info("Catalogueing Scripts On The System")
sm.catalogue_local_scripts(sqlite_manager, self._script_dir,
self._logger)
# create process for listening for node connections
# READ through parent_pipe, WRITE through child_pipe
try:
self._logger.info("Now Creating Pipe")
parent_pipe, child_pipe = Pipe()
self._logger.info("Now Creating NodeClientProcess Class")
# node_listener = NodeListenerProcess(to_parent_pipe, to_child_pipe, self._config)
self._logger.info("Now Creating Process With BootStrapper")
self._node_process = Process(target=bootstrapper,
args=(NodeClientProcess,
(child_pipe, self._config,
logutils.logging_queue)))
self._logger.info("Now Starting Process")
self._node_process.start()
self._logger.info("Node Process Has Started Running")
except Exception as e:
self._logger.exception(
"An Exception Was Thrown Starting The Node Listener Process")
self._logger.error("Later - An Exception Was Thrown")
return
# create process for listening for http connections
# start logging thread
l_thread = logutils.start_logging_thread()
rc = None
while rc != win32event.WAIT_OBJECT_0:
self._logger.info("Service Is Now Running")
# hang for 1 minute or until service is stopped - whichever comes first
rc = win32event.WaitForSingleObject(self.hWaitStop,
(1 * 60 * 1000))
self._node_process.terminate()
def GETScriptOfGuid(script_guid):
self.logger.info("Fetching Script Of Guid: " + script_guid)
try:
uuid.UUID(script_guid, version=4)
except:
abort(400, "The Passed In Script Guid Is Invalid")
uuid_script_guid = uuid.UUID(script_guid)
sql_manager = SQLiteManager(self._config, self.logger)
all_scripts = sql_manager.getAllScripts()
for script in all_scripts:
if script.guid == uuid_script_guid:
script_dict = script.toDictionary()
script_dict.pop("file_path", None)
sql_manager.closeEverything()
return jsonify(script_dict)
sql_manager.closeEverything()
return abort(404)
def GETNodeOfGuid(node_guid):
self.logger.info("Fetching Node Of Guid: " + node_guid)
try:
uuid.UUID(node_guid, version=4)
except:
abort(400, "The Passed In Node Guid Is Invalid")
uuid_node_guid = uuid.UUID(node_guid)
sql_manager = SQLiteManager(self._config, self.logger)
all_nodes = sql_manager.getAllNodes()
for node in all_nodes:
if node.guid == uuid_node_guid:
node_dict = node.toDictionary()
node_dict.pop("ip", None)
sql_manager.closeEverything()
return jsonify(node_dict)
sql_manager.closeEverything()
return abort(404)
def SvcStop(self):
self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
win32event.SetEvent(self.hWaitStop)
self.shutdown_occurring = True
self._logger.info("Service Is Stopping")
self._logger.info("Fetching All Nodes To Send Restart Requests")
# get list of all the nodes
sql_manager = SQLiteManager(self._config, self._logger)
all_nodes = sql_manager.getAllNodes()
self._logger.info("Nodes Fetched. Parsing")
self._logger.info("Parsing Complete")
self._logger.info("Nodes Fetched. Now Sending")
# send message to each node to disconnect, sleep and then start infinite reconnect attempts
for node in all_nodes:
action = dict()
action['command'] = "SYS"
action['from'] = "MASTER"
action['to'] = "NODE"
action['params'] = "RESTART"
action['rawdata'] = (str(node.guid),)
self.sendMessageToNodeProcess(action)
self._logger.info("Now Waiting For Nodes To Disconnect")
# wait for all the nodes to disconnect via checking the db
while len(sql_manager.getAllNodes()) > 0:
time.sleep(2)
pass
self._logger.info("Parse Complete. Closing Connection")
sql_manager.closeEverything()
# now it is safe to terminate
self.shutdown_processing_complete = True
self._logger.info("Shutdown Processing Completed")
def main(self):
self._logger.info("Service Is Initializing...")
# setup database
sqlite_manager = SQLiteManager(self._config, self._logger)
self._logger.info("Master Role Detected. Setting Up Service For Master Role")
# catalogue all the scripts in the system
self._logger.info("Catalogueing Engines On The System")
sm.catalogue_local_engines(sqlite_manager, self._logger)
self._logger.info("Catalogueing Scripts On The System")
sm.catalogue_local_scripts(sqlite_manager, self._script_dir, self._logger)
# create process for listening for terminal connections
try:
self._logger.info("Now Creating Pipe For Terminal Process")
terminal_parent_pipe, terminal_child_pipe = Pipe()
self.terminal_parent_pipe = terminal_parent_pipe
self._logger.info("Now Creating TerminalListenerProcess Class")
self._logger.info("Now Creating Process With Boostrapper")
self._terminal_process = Process(target=bootstrapper, args=(TerminalListenerProcess, (terminal_child_pipe,
self._config,
logutils.logging_queue)))
self._logger.info("Now Starting Process")
self._terminal_process.start()
self._logger.info("Termina Process Has Started Running")
except Exception as e:
self._logger.exception("An Exception Was Thrown Starting The Node Listener Process")
self._logger.error("Later - An Exception Was Thrown")
return
# create process for listening for node connections
# READ through parent_pipe, WRITE through child_pipe
try:
self._logger.info("Now Creating Pipe For Node Process")
node_parent_pipe, node_child_pipe = Pipe()
self.node_parent_pipe = node_parent_pipe
self._logger.info("Now Creating NodeListenerProcess Class")
#node_listener = NodeListenerProcess(to_parent_pipe, to_child_pipe, self._config)
self._logger.info("Now Creating Process With BootStrapper")
self._node_process = Process(target=bootstrapper, args=(NodeListenerProcess,(node_child_pipe, self._config,
logutils.logging_queue)))
self._logger.info("Now Starting Process")
self._node_process.start()
self._logger.info("Http Process Has Started Running")
except Exception as e:
self._logger.exception("An Exception Was Thrown Starting The Node Listener Process")
self._logger.error("Later - An Exception Was Thrown")
return
# create process for listening for http connections
try:
self._logger.info("Now Creating Pipe For Http Process")
http_parent_pipe, http_child_pipe = Pipe()
self.http_parent_pipe = http_parent_pipe
self._logger.info("Now Creating HttpListenerProcess Class")
self._logger.info("Now Creating Process With Bootstrapper")
self._http_process = Process(target=bootstrapper, args=(HttpListenerProcess, (http_child_pipe, self._config,
logutils.logging_queue)))
self._logger.info("Now Starting Http Process")
self._http_process.start()
self._logger.info("Http Process Has Started Running")
except Exception as e:
self._logger.exception("An Exception Was Thrown Starting The Http Listener Process")
self._logger.error("Later - An Exception Was Thrown")
return
# spawn threads to handle listening for commands from these three sources
self._logger.info("Launching Pipe Listening Thread For Terminal Process")
t_thread = threading.Thread(target=pipe_recv_handler,
args=(self, terminal_parent_pipe))
t_thread.daemon = True
t_thread.start()
self._logger.info("Launching Pipe Listening Thread For Node Process")
n_thread = threading.Thread(target=pipe_recv_handler,
args=(self, node_parent_pipe))
n_thread.daemon = True
n_thread.start()
self._logger.info("Launching Pipe Listening Thread For Http Process")
h_thread = threading.Thread(target=pipe_recv_handler,
args=(self, http_parent_pipe))
h_thread.daemon = True
h_thread.start()
# spawn logging thread
l_thread = logutils.start_logging_thread()
rc = None
while rc != win32event.WAIT_OBJECT_0:
self._logger.info("Service Is Now Running")
# hang for 1 minute or until service is stopped - whichever comes first
rc = win32event.WaitForSingleObject(self.hWaitStop, (1 * 60 * 1000))
self._logger.info("Service Shutdown Detected In Main Loop. Waiting For Shutdown Process To Complete")
# don't terminate processes until all of the shutdown procedure has completed
while not self.shutdown_processing_complete:
# loop until its done
pass
self._logger.info("Shutdown Process Completed. Terminating Other Processes")
# now temrinate processes
self._node_process.terminate()
self._terminal_process.terminate()
self._http_process.terminate()
self._logger.info("Main Loop Termination Completed. Terminating")
def socket_recv_handler(node_listener_process, logger, node_socket, child_pipe):
logger.info("Starting Socket Receive Handler")
while True:
sql_manager = SQLiteManager(node_listener_process._config, node_listener_process.logger)
try:
valid_message_received = False
raw_message: bytes = b''
while not valid_message_received:
base64_encrypted_bytes = node_socket.recv(4096)
raw_message += base64_encrypted_bytes
if len(raw_message) > 0:
if raw_message[:1] == b'{' and raw_message[len(raw_message) - 1:] == b'}':
valid_message_received = True
raw_message = raw_message[1:len(raw_message)-1]
# find the node belonging to this socket
address = node_listener_process.socketmap2portip[node_socket]
ip, port = address
node = sql_manager.getNodeOfIpAndPort(ip, port)
# find the aes key for this node
key = sql_manager.getKeyOfGuid(node.key_guid)
# decrypt the key
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(key.key.encode(),
node_listener_process.master_private_key,
node_listener_process.private_key_password)
command = vh.decrypt_base64_bytes_with_aes_key_to_string(raw_message, aes_key)
logger.info("Command Received From Socket: " + str(command))
command_dict = json.loads(command)
if command_dict["command"] == "SYS" and command_dict["to"] == "MASTER" \
and command_dict["param"] == "CONN.CLOSE":
# this means the remote node is gracefully exiting. We should treate this as if it disconnected
logger.info("Graceful Disconnect From Node Detected. Throwing Exception To Trigger Disconnection")
os_error = OSError()
os_error.errno = errno.ECONNRESET
raise os_error
else:
child_pipe.send(command_dict)
except OSError as se:
if se.errno == errno.ECONNRESET:
logger.info("Connection Reset Detected. Failed To Receive Message From Node. Node Does Not Exist")
# cleanup socket information on our side
try:
node_socket.shutdown(socket.SHUT_RDWR)
except:
pass
try:
node_socket.close()
except:
pass
logger.info("Removing Socket From Mappings")
address = node_listener_process.socketmap2portip[node_socket]
ip, port = address
node_listener_process.portipmap2socket.pop(ip+":"+str(port), None)
node_listener_process.connections.pop(node_socket.fileno(), None)
logger.info("Removing Socket From Database")
sql_manager = SQLiteManager(node_listener_process._config, node_listener_process.logger)
node = sql_manager.getNodeOfIpAndPort(ip, port)
sql_manager.deleteKeyOfGuid(node.key_guid)
sql_manager.deleteNodeOfGuid(node.guid)
logger.info("Removing Socket From Last Mapping")
node_listener_process.socketmap2portip.pop(node_socket, None)
logger.info("Diconnection Process Of Node Complete. Terminating Socket Receive Handler Thread")
break
else:
logger.info("Other Error Thrown")
logger.info(se.strerror)
finally:
sql_manager.closeEverything()
def forwardCommandToAppropriateNode(self, command, node_guid: str)->bool:
self.logger.info("Now Attempting Forwarding Command To Appropriate Node")
self.logger.info(("Searching For Socket Matching Node Guid: " + str(node_guid)))
sql_manager = SQLiteManager(self._config, self.logger)
node = sql_manager.getNodeOfGuid(str(node_guid))
self.logger.info("Search Mapped To IP: " + node.ip + " And PORT: " + node.port)
node_socket2 = self.portipmap2socket.get(node.ip + ":" + str(node.port), None)
self.logger.info("Socket: " + str(node_socket2))
if node_socket2 is None:
self.logger.info("WARNING: IP and Port Mapping Did Not Resolve To A Socket. Can't Forward Command")
return False
serialized_command = json.dumps(command)
self.logger.info("Serialized Command: >" + str(serialized_command) + "<")
try:
key = sql_manager.getKeyOfGuid(node.key_guid)
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(key.key.encode(),
self.master_private_key,
self.private_key_password)
base64_encrypted_bytes = vh.encrypt_string_with_aes_key_to_base64_bytes(serialized_command,
aes_key)
base64_encrypted_bytes = b'{' + base64_encrypted_bytes + b'}'
node_socket2.send(base64_encrypted_bytes)
self.logger.info("Serialized Message Sent")
sql_manager.closeEverything() # can't use sql_manager after this
return True
except error as se:
if se.errno == errno.ECONNRESET:
self.logger.info("Connection Reset Detected. Failed To Send Message To Node. Node Does Not Exist")
# cleanup socket information on our side
try:
node_socket2.shutdown(socket.SHUT_RDWR)
except:
pass
try:
node_socket2.close()
except:
pass
self.logger.info("Removing Socket From Mappings")
self.portipmap2socket.pop(node.ip+":"+str(node.port), None)
self.connections.pop(node_socket2, None)
self.logger.info("Removing Socket From Database")
sql_manager.deleteKeyOfGuid(node.key_guid)
sql_manager.deleteNodeOfGuid(node.guid)
sql_manager.closeEverything() # can't use sql_manager after this
self.logger.info("Removing Socket From Last Mapping")
self.socketmap2portip.pop(node_socket2, None)
# return False to tell caller
return False
else:
self.logger.info("ERROR: Unknown Socket Error Occured In Node Listener Process. Error Code " + str(se.errno))
self.logger.info("Error Details: " + se.strerror)
self.logger.info(se)
finally:
sql_manager.closeEverything() # can't use sql_manager after this
class NodeListenerProcess:
_sql_manager = None
child_pipe = None
_port = None
logger = None
master_private_key = None
master_public_key = None
private_key_password = None
aes_key = None
connections = dict()
portipmap2socket = dict()
socketmap2portip = dict()
_config = None
_all_to_read = []
def __init__(self, initialization_tuple):
child_pipe, config, logging_queue = initialization_tuple
self.child_pipe = child_pipe
self._config = config
self._port = config["NODELISTENER"]["port"]
self._bind_ip = config["NODELISTENER"]["bind_ip"]
self._log_dir = config["LOGGING"]["log_dir"]
self.private_key_password = config["DEFAULT"]["private_key_password"]
qh = logging.handlers.QueueHandler(logging_queue)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(qh)
self.logger = logging.getLogger("NodeListenerProcessLogger")
self.logger.setLevel(logging.DEBUG)
self.logger.info("NodeListenerProcess Inialized. Creating Connection To SQL DB")
self._sql_manager = SQLiteManager(config, self.logger)
self.logger.info("Connection Complete")
def forwardCommandToAppropriateNode(self, command, node_guid: str)->bool:
self.logger.info("Now Attempting Forwarding Command To Appropriate Node")
self.logger.info(("Searching For Socket Matching Node Guid: " + str(node_guid)))
sql_manager = SQLiteManager(self._config, self.logger)
node = sql_manager.getNodeOfGuid(str(node_guid))
self.logger.info("Search Mapped To IP: " + node.ip + " And PORT: " + node.port)
node_socket2 = self.portipmap2socket.get(node.ip + ":" + str(node.port), None)
self.logger.info("Socket: " + str(node_socket2))
if node_socket2 is None:
self.logger.info("WARNING: IP and Port Mapping Did Not Resolve To A Socket. Can't Forward Command")
return False
serialized_command = json.dumps(command)
self.logger.info("Serialized Command: >" + str(serialized_command) + "<")
try:
key = sql_manager.getKeyOfGuid(node.key_guid)
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(key.key.encode(),
self.master_private_key,
self.private_key_password)
base64_encrypted_bytes = vh.encrypt_string_with_aes_key_to_base64_bytes(serialized_command,
aes_key)
base64_encrypted_bytes = b'{' + base64_encrypted_bytes + b'}'
node_socket2.send(base64_encrypted_bytes)
self.logger.info("Serialized Message Sent")
sql_manager.closeEverything() # can't use sql_manager after this
return True
except error as se:
if se.errno == errno.ECONNRESET:
self.logger.info("Connection Reset Detected. Failed To Send Message To Node. Node Does Not Exist")
# cleanup socket information on our side
try:
node_socket2.shutdown(socket.SHUT_RDWR)
except:
pass
try:
node_socket2.close()
except:
pass
self.logger.info("Removing Socket From Mappings")
self.portipmap2socket.pop(node.ip+":"+str(node.port), None)
self.connections.pop(node_socket2, None)
self.logger.info("Removing Socket From Database")
sql_manager.deleteKeyOfGuid(node.key_guid)
sql_manager.deleteNodeOfGuid(node.guid)
sql_manager.closeEverything() # can't use sql_manager after this
self.logger.info("Removing Socket From Last Mapping")
self.socketmap2portip.pop(node_socket2, None)
# return False to tell caller
return False
else:
self.logger.info("ERROR: Unknown Socket Error Occured In Node Listener Process. Error Code " + str(se.errno))
self.logger.info("Error Details: " + se.strerror)
self.logger.info(se)
finally:
sql_manager.closeEverything() # can't use sql_manager after this
def start(self):
try:
self.logger.info("Fetching/Generating Public And Private Keys For Node Communication")
private_key = self._sql_manager.getKeyOfName("master-me.key.private")
public_key = self._sql_manager.getKeyOfName("master-me.key.public")
# FIXME: There is no proper handling IF one of the keys exists and the other doesn't!
if private_key is None or public_key is None:
self.master_private_key = vh.generate_private_key(self.private_key_password)
self.master_public_key = vh.generate_public_key(self.master_private_key, self.private_key_password)
private_key = Key()
private_key.name = "master-me.key.private"
private_key.key = self.master_private_key
self._sql_manager.insertKey(private_key)
public_key = Key()
public_key.name = "master-me.key.public"
public_key.key = self.master_public_key
self._sql_manager.insertKey(public_key)
else:
self.master_private_key = private_key.key
self.master_public_key = public_key.key
self.logger.info("Launching Pipe Listening Thread")
t = threading.Thread(target=pipe_recv_handler, args=(self, self.logger, self.child_pipe))
t.daemon = True
t.start()
self.logger.info("Initializing Listener Socket")
# startup the listening socket
#try:
listener_socket = socket(AF_INET, SOCK_STREAM)
listener_socket.bind((self._bind_ip, int(self._port)))
listener_socket.listen(10)
self.logger.info("Storing Socket Information")
# store listening socket fd
self.connections[listener_socket.fileno()] = listener_socket
self.logger.info("Now Entering Connection Acceptance Loop")
while True:
node_socket, address = listener_socket.accept()
self.logger.info("New Connection Detected. Processing And Adding To System")
self.logger.info("Enabling Keep Alive Policy In New Connection")
node_socket.ioctl(SIO_KEEPALIVE_VALS, (1, 10000, 3000))
self.logger.info("Updating Memory Mappings For New Connection")
# add mapping table records
self.connections[node_socket.fileno()] = node_socket
self.socketmap2portip[node_socket] = address
client_ip, client_port = address
self.portipmap2socket[client_ip + ":" + str(client_port)] = node_socket
self.logger.info("Passing Security Keys To The New Node")
#send the public key
node_socket.send(self.master_public_key.encode())
#get the encrypted aes key
aes_key_encrypted = node_socket.recv(4096)
# verify this key is valid
try:
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(aes_key_encrypted,
self.master_private_key,
self.private_key_password)
decoded = aes_key_encrypted.decode('utf-8')
except:
self.logger.exception("Incoming AES Key Is Invalid Or Failed Validation. Assuming Connection"
"Is Invalid. Closing Connection")
node_socket.close()
continue
self.logger.info("Making Ping Request To Retrieve Node Information")
# pass a command to the node to fetch ping information and get the node name
action = dict()
action['command'] = "GET"
action['from'] = "LISTENER"
action['to'] = "NODE"
action['params'] = "PING"
serialized_command = json.dumps(action)
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(aes_key_encrypted,
self.master_private_key,
self.private_key_password)
base64_encrypted_bytes = vh.encrypt_string_with_aes_key_to_base64_bytes(serialized_command,
aes_key)
node_socket.send(b'{' + base64_encrypted_bytes + b'}')
self.logger.info("Request Sent")
valid_message_received = False
full_encrypted_bytes: bytes = b''
while not valid_message_received:
self.logger.info("Getting Segment of Response")
encrypted_bytes = node_socket.recv(4096)
full_encrypted_bytes += encrypted_bytes
if len(full_encrypted_bytes) > 0:
if full_encrypted_bytes[:1] == b'{' and full_encrypted_bytes[len(full_encrypted_bytes) - 1:] == b'}':
valid_message_received = True
self.logger.info("Full Message Segment Parsed. Now Processing")
# place back into encrypted_bytes the trimmed and fixed message
encrypted_bytes = full_encrypted_bytes[1:len(full_encrypted_bytes)]
command = vh.decrypt_base64_bytes_with_aes_key_to_string(encrypted_bytes, aes_key)
command_dict = json.loads(command)
if command_dict["command"] == "ERROR":
# connection had a fatal error
self.logger.info("Fatal Error Trying To Ping Recently Established Node. Failed To Complete "
"Connection Sequence. Connection Will Be Terminated")
self.logger.info(command)
node_socket.close()
continue
self.logger.info("Adding Key To DB")
# add the key to our db
key = Key()
key.key = aes_key_encrypted.decode(
'utf-8') # note this key is encrypted with our public key and then base64 encoded
key.name = "node.key.aes"
key = self._sql_manager.insertKey(key)
node_name = command_dict['rawdata']['node-name']
self.logger.info("Adding Node To DB")
# add this node to our db
node = Node()
node.ip = client_ip
node.port = client_port
node.name = node_name
node.key_guid = key.guid
node.state = "UP"
self._sql_manager.insertNode(node)
self.logger.info("New Connection Establishment Complete")
self.logger.info("Now Spawning Processing Thread To Handle Future Reads By This Socket")
self.logger.info("Launching Socket Listening Thread")
t = threading.Thread(target=socket_recv_handler, args=(self,
self.logger,
node_socket,
self.child_pipe))
t.daemon = True
t.start()
self.logger.info("Processing Of New Connection Complete")
except Exception as e:
self.logger.exception("Fatal / Unknown Error Processing For Node Listener. Node Listener Process"
" Will Terminate")
class NodeClientProcess:
_node_private_key = None
_node_public_key = None
_master_public_key = None
_node_aes_key = None
_script_dir = None
_node_name = None
_client_socket = None
_config = None
failed_initializing = False
def __init__(self, initialization_tuple):
child_pipe, config, logging_queue = initialization_tuple
self.child_pipe = child_pipe
self._config = config
self._master_host = config["DEFAULT"]["master_domain"]
self._master_port = config["DEFAULT"]["master_port"]
self._log_dir = config["LOGGING"]["log_dir"]
self._root_dir = config["DEFAULT"]["root_dir"]
self._node_name = config["DEFAULT"].get("name", "node")
self._script_dir = config["DEFAULT"].get("scripts_dir",
self._root_dir + "/scripts")
self._private_key_password = config["DEFAULT"]["private_key_password"]
# setup logging
qh = logging.handlers.QueueHandler(logging_queue)
root = logging.getLogger()
root.setLevel(logging.DEBUG)
root.addHandler(qh)
self.logger = logging.getLogger("NodeClientProcessLogger")
self.logger.setLevel(logging.DEBUG)
self.logger.info(
"NodeListenerProcess Inialized. Creating Connection To SQL DB")
self._sql_manager = SQLiteManager(config, self.logger)
self.logger.info("Connection Complete")
def _send_message(self, message: str, encrypt_with_key=None) -> int:
try:
if encrypt_with_key is not None:
node_private_key, node_private_key_password, node_aes_key = encrypt_with_key
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(
node_aes_key, node_private_key, node_private_key_password)
base64_encrypted_bytes = vh.encrypt_string_with_aes_key_to_base64_bytes(
message, aes_key)
base64_encrypted_bytes = b'{' + base64_encrypted_bytes + b'}'
return self._client_socket.send(base64_encrypted_bytes)
else:
return self._client_socket.send(message.encode())
except OSError as se:
if se.errno == errno.ECONNRESET:
self.logger.info(
"Connection Reset Detected While Trying To Send Message. Attempting Connection Reestablishment"
)
# close socket
try:
self._client_socket.shutdown(socket.SHUT_RDWR)
except:
pass
try:
self._client_socket.close()
except:
pass
reconnect_succeeded = self.execute_connect_to_host_procedure()
if not reconnect_succeeded:
self.logger.fatal(
"Connection Reestablishment Failed. Not Bothering Again. Terminating Process"
)
action = dict()
action['command'] = "SYS"
action['from'] = "CLIENT"
action['to'] = "NODE"
action['params'] = "SHUTDOWN"
self.child_pipe.send(action)
exit()
return 0
else:
return self._send_message(
message, encrypt_with_key=encrypt_with_key)
else:
self.logger.info(
"An OSError Was Thrown While Trying To Send Message")
self.logger.info(se)
return 0
def _recv_message(self, buffer_size, decrypt_with_key_pass=None):
try:
raw_message: bytes = b''
valid_message_received = False
if decrypt_with_key_pass is not None:
while not valid_message_received:
base64_encrypted_bytes = self._client_socket.recv(
buffer_size)
raw_message += base64_encrypted_bytes
if len(raw_message) > 0:
if raw_message[:1] == b'{' and raw_message[
len(raw_message) - 1:] == b'}':
valid_message_received = True
raw_message = raw_message[1:len(raw_message) - 1]
node_private_key, node_private_key_password, node_aes_key = decrypt_with_key_pass
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(
node_aes_key, node_private_key, node_private_key_password)
message = vh.decrypt_base64_bytes_with_aes_key_to_string(
raw_message, aes_key)
return message
else:
return self._client_socket.recv(buffer_size).decode('utf8')
except OSError as se:
if se.errno == errno.ECONNRESET:
self.logger.info(
"Connection Reset Detected While Trying To Receive Message. Attempting Connection Reestablishment"
)
# close socket
try:
self._client_socket.shutdown(socket.SHUT_RDWR)
except:
pass
try:
self._client_socket.close()
except:
pass
reconnect_succeeded = self.execute_connect_to_host_procedure()
if not reconnect_succeeded:
self.logger.fatal(
"Connection Reestablishment Failed. Not Bothering Again. Terminating Process"
)
return None
else:
self.logger.info(
"An OSError Was Thrown While Trying To Receive Message")
self.logger.info(se)
return None
def execute_connect_to_host_procedure(self) -> bool:
try:
address_results = getaddrinfo(self._master_host,
int(self._master_port))
master_ip = address_results[0][4][0]
self._client_socket = socket(AF_INET, SOCK_STREAM)
self._client_socket.connect((master_ip, int(self._master_port)))
except:
self.logger.exception(
"Resolution Of Master Domain Or Connection Failed. Aborting Processing"
)
self.failed_initializing = True
return False
try:
self.logger.info(
"Connection Established With Master. Securing Connection With Keys"
)
self._master_public_key = self._recv_message(2048)
if self._master_public_key is None:
self.logger.fatal(
"Failed To Receive Master Node Public Key. Terminating")
# TODO: Pass Message to node process to shut service down
action = dict()
action['command'] = "SYS"
action['from'] = "CLIENT"
action['to'] = "NODE"
action['params'] = "SHUTDOWN"
self.child_pipe.send(action)
exit()
aes_key = vh.decrypt_base64_bytes_with_private_key_to_bytes(
self._node_aes_key, self._node_private_key,
self._private_key_password)
encrypted_aes_key = vh.encrypt_bytes_with_public_key_to_base64_bytes(
aes_key, self._master_public_key)
self._send_message(encrypted_aes_key.decode('utf-8'))
self.logger.info("Key Received From Master")
self.logger.info(self._master_public_key)
self.logger.info("Connection Secured")
except:
self.logger.exception(
"Connection With Master Node Failed. Aborting Processing")
self.failed_initializing = True
return False
return True
def start(self):
command_dict = None
try:
self.logger.info("Configuring Local Keys")
# generate our RSA Private And Public Keys
private_key = self._sql_manager.getKeyOfName("node-me.key.private")
public_key = self._sql_manager.getKeyOfName("node-me.key.public")
found_aes_key = self._sql_manager.getKeyOfName("node-me.key.aes")
# FIXME: There is no proper handling IF one of the keys exists and the other doesn't!
if private_key is None or public_key is None or found_aes_key is None:
self.logger.info(
"Keys Have Not Been Generated Before On This Node. This May Take Some Time..."
)
self._node_private_key = vh.generate_private_key(
self._private_key_password)
self._node_public_key = vh.generate_public_key(
self._node_private_key, self._private_key_password)
new_aes_key = vh.generate_aes_key(self._private_key_password)
# our aes key is stored encrypted with our public key
self._node_aes_key = vh.encrypt_bytes_with_public_key_to_base64_bytes(
new_aes_key, self._node_public_key)
private_key = Key()
private_key.name = "node-me.key.private"
private_key.key = self._node_private_key
self._sql_manager.insertKey(private_key)
public_key = Key()
public_key.name = "node-me.key.public"
public_key.key = self._node_public_key
self._sql_manager.insertKey(public_key)
key = Key()
key.key = self._node_aes_key.decode(
'utf-8'
) # note this key is encrypted with our public key and then base64 encoded
key.name = "node-me.key.aes"
self._sql_manager.insertKey(key)
else:
self._node_private_key = private_key.key
self._node_public_key = public_key.key
self._node_aes_key = found_aes_key.key.encode()
self.logger.info("Local Key Generation Complete")
self.logger.info("Initializing Socket To Master")
self.execute_connect_to_host_procedure()
while True:
self.logger.info("Reading Command From Socket")
command = self._recv_message(
4096,
decrypt_with_key_pass=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
if command is None:
self.logger.error(
"Failed To Receive Command. Disconnection From Master Likely Occurred. Can't "
"Process Command")
action = dict()
action['command'] = "SYS"
action['from'] = "CLIENT"
action['to'] = "NODE"
action['params'] = "SHUTDOWN"
self.child_pipe.send(action)
exit()
command_dict = json.loads(command)
self.logger.info("COMMAND RECEIVED")
self.logger.info(command)
try:
if command_dict["command"] == "GET" and command_dict[
"params"] == "SCRIPTS":
self.logger.info(
"Fetch Node Scripts Request Detected. Executing")
response = taskrunner.fetch_node_scripts(
self._sql_manager, command_dict, self.logger)
self.logger.info(
"Fetched Data. Now Serializing For Response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "EXEC" and command_dict[
"params"] == "SCAN.SCRIPTS":
self.logger.info(
"Scan Scripts Request Detected. Executing")
sm.catalogue_local_scripts(self._sql_manager,
self._script_dir,
self.logger)
response = taskrunner.fetch_node_scripts(
self._sql_manager, command_dict, self.logger)
self.logger.info(
"Fetched Data. Now Serializing For Response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "GET" and command_dict[
"params"] == "PING":
self.logger.info(
"Get Ping Request Detected. Executing")
response = taskrunner.get_ping_info(
command_dict, self._config)
self.logger.info(
"Fetched Data. Now Serializing For Response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "CREATE" and command_dict[
"params"] == "DEPLOYMENT":
self.logger.info(
"Create Deployment Request Detected. Executing")
response = taskrunner.create_deployment(
self._sql_manager, command_dict, self.logger)
self.logger.info(
"Fetched Data. Now Serializing For Response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "GET" and command_dict[
"params"] == "DEPLOYMENTS":
self.logger.info(
"Fetch Node Deployments Request Detected. Executing"
)
response = taskrunner.fetch_node_deployments(
self._sql_manager, command_dict, self.logger)
self.logger.info(
"Fetched Data. Now Serializing For response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "EXEC" and command_dict[
"params"] == "DEPLOYMENTS.EXECUTE":
self.logger.info(
"Executing Deployment On Node Request Detected. Executing"
)
response = taskrunner.execute_deployment_on_node(
self._sql_manager, command_dict, self.logger)
self.logger.info(
"Fetched Data. Now Serializing For response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "EXEC" and command_dict[
"params"] == "SCRIPTS.EXECUTE":
self.logger.info(
"Executing Script On Node Request Detected. Executing"
)
response = taskrunner.execute_script_on_node(
self._sql_manager, command_dict, self.logger)
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
elif command_dict["command"] == "MIG":
self.logger.info(
"Script Migration Request Received. Importing Script"
)
response = taskrunner.migrate(self._root_dir,
self._sql_manager,
command_dict,
self.logger)
self.logger.info(
"Fetched Data. Now Serializing For Response")
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
elif command_dict["command"] == "SYS" and command_dict[
"params"] == "RESTART":
self.logger.info(
"Master Node Restart Request Received. Disconnecting and Starting "
"Reconnect Loop")
# send back a clean exit
response = dict()
response["to"] = "MASTER"
response["from"] = command_dict["to"]
response["command"] = "SYS"
response["param"] = "CONN.CLOSE"
response["rawdata"] = command_dict["rawdata"]
serialized_data = json.dumps(response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
self.logger.info("Response Sent")
# disconnect from master. Sleep for 2 minutes, then start reconnecting
try:
self._client_socket.shutdown(socket.SHUT_RDWR)
except:
pass
try:
self._client_socket.close()
except:
pass
self.logger.info(
"Diconnect From Master Node Complete. Sleeping For 2 Minutes"
)
time.sleep(120)
self.logger.info(
"Sleep Period Completed. Starting Reconnection")
attempt_count = 0
while not self.execute_connect_to_host_procedure():
attempt_count += 1
self.logger.info(
"Reconnection Failed. Sleeping For 30 Seconds and Trying Again "
+ "(Attempts: " + str(attempt_count) + ")")
time.sleep(30)
self.logger.info(
"Reconnection Successful. SYS.RESTART Complete")
else:
self.logger.info(
"Received Command Has Not Been Configured A Handling. Cannot Process Command"
)
error_response = dict()
error_response['command'] = 'ERROR'
error_response['from'] = 'node_client'
error_response['to'] = command_dict['from']
error_response['params'] = "Command: " + command_dict['command'] + " From: " + command_dict['from'] + \
" To: " + command_dict['to']
error_response['rawdata'] = (
"Received Command Has No Mapping On This Node. Cannot Process Command",
command_dict)
serialized_data = json.dumps(error_response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
except Exception as e:
self.logger.exception(
"Unexpected Error Thrown While Processing a Request.")
if command_dict is not None:
error_response = dict()
error_response['command'] = 'ERROR'
error_response['from'] = 'node_client'
error_response['to'] = command_dict['from']
error_response['params'] = "Command: " + command_dict['command'] + " From: " + command_dict['from'] + \
" To: " + command_dict['to']
error_response[
'rawdata'] = "UnExpected Error Executing Request: " + str(
e)
serialized_data = json.dumps(error_response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))
except Exception as e:
self.logger.exception("Fatal Error Processing For Node Client")
if command_dict is not None:
error_response = dict()
error_response['command'] = 'ERROR'
error_response['from'] = 'node_client'
error_response['to'] = command_dict['from']
error_response['params'] = "Command: " + command_dict['command'] + " From: " + command_dict['from'] + \
" To: " + command_dict['to']
error_response['rawdata'] = "UnExpected Error: " + str(e) + " WARNING: Node Has Likely Terminated From " \
"This Event Or Is In A Broken State. Restart " \
"To Recover"
serialized_data = json.dumps(error_response)
self._send_message(
str(serialized_data),
encrypt_with_key=(self._node_private_key,
self._private_key_password,
self._node_aes_key))