class Engine:
def __init__(self, local_ip, local_port, remote_ip, remote_port):
self.recommender = MLEngine(D2LogisticRegression())
self.local_ip = local_ip
self.local_port = local_port
self.remote_ip = remote_ip
self.remote_port = remote_port
self.heart_beater = HeartBeat("54.186.108.36:8888", remote_ip, remote_port)
self.task_handler = TaskHandler(remote_ip, remote_port)
self.client_thread = threading.Thread(target=self.start_task_creator, args = ())
self.remote_thread = threading.Thread(target=self.start_remote_handler, args = ())
self.heart_beater.start()
self.client_thread.start()
self.remote_thread.start()
self.client_thread.join()
self.remote_thread.join()
self.task_handler.join()
self.heart_beater.stop()
def start_task_creator(self):
address = (self.local_ip, int(self.local_port))
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.bind(address)
while True:
print("root receive new task [start]")
content, addr = server_socket.recvfrom(2048)
command = json.loads(content.decode("utf-8"))
radiant = [i for i in command["radiant"] if i != -1 and i!=77]
dire = [i for i in command["dire"] if i != -1 and i!=77]
command["dire"] = dire
command["radiant"] = radiant
responses = self.task_handler.launch_task(Task(command, server_socket, addr))
print("root receive new task [end]")
#server_socket.sendto("hello c#".encode("utf-8"), addr)
def process_request(self, request):
if request["camp"] == 0:
return self.recommender.recommend(request["radiant"], request["dire"])
else:
return self.recommender.recommend(request["dire"], request["radiant"])
def start_remote_handler(self):
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # UDP
sock.bind((self.remote_ip, int(self.remote_port)))
result = {}
while True:
data, addr = sock.recvfrom(4096)
message = data.decode("utf-8")
print("receive remote request: {}",message)
request = json.loads(message)
result["candidates"] = self.process_request(request)
result["ip"] = self.remote_ip
result["port"] = self.remote_port
sock.sendto(json.dumps(result).encode("utf-8"), addr)
print("send remote response: {}",json.dumps(result))
def runIDS(verbose=False):
print("Starting IDS...")
try:
# Create log file if it does not exist.
if not os.path.exists(r'logs\idslogs\ids.log'):
file = open(os.path.join(r'logs\idslogs', 'ids.log'), 'w')
file.close()
logging.basicConfig(filename=os.path.join(r'logs\idslogs', 'ids.log'),
level=logging.INFO)
csvloader = CSVFlowLoader(os.path.join(CSVFILEPATH, csvfilename))
mlengine = MLEngine(MODEL_FILEPATH, DATACLEAN_PIPELINE_FILEPATH,
DATAPREP_PIPELINE_FILEPATH)
while True:
for flowline in csvloader.tailFile():
csValsArray = [list(flowline.split(","))]
csValsDF = pd.DataFrame(csValsArray, columns=columns)
# Actual detection and printing results out in stdout.
if mlengine.predict(csValsDF)[0] == PredictLabel.ANOMALY.value:
print("ANOMALY: %s" % (parsePredictionDF(csValsDF)))
logging.info("ANOMALY: %s" % (parsePredictionDF(csValsDF)))
else:
if verbose:
print("BENIGN: %s" % (parsePredictionDF(csValsDF)))
except KeyboardInterrupt:
print("Exiting...")
csvloader.destroy()
def test_predict(self):
columns = constants.COLUMNS
mlengine = MLEngine()
# resultSet = [-1,1]
# inputData = [[49188,4,2,0,12,0,6,6,6,0,0,0,0,0,14000,3500,4,0,4,4,4,4,0,4,4,0,0,0,0,0,0,0,0,0,40,0,500000,0,6,6,6,0,0,0,0,0,0,1,1,0,0,0,9,6,0,40,0,0,0,0,0,0,2,12,0,0,329,-1,1,20,0,0,0,0,0,0,0,0]]
inputData = [[
'192.168.91.225-13.107.4.52-65026-80-6', '192.168.91.225', 65026,
'13.107.4.52', 80, 6, '30/09/2019 07:34:49 PM', 19002, 2, 5, 111.0,
1009.0, 111.0, 0.0, 55.5, 78.48885271170677, 1009.0, 0.0, 201.8,
451.2385178594576, 58941.164087990735, 368.3822755499421, 3167.0,
3013.9242857112386, 7587.0, 135.0, 543.0, 543.0, 0.0, 543.0, 543.0,
19002.0, 4750.5, 2930.5069527301926, 8265.0, 1358.0, 0, 0, 0, 0,
40, 124, 105.25207872855488, 263.1301968213872, 0.0, 1009.0, 140.0,
353.2708390375375, 124800.28571428571, 0, 1, 0, 0, 0, 0, 0, 0, 2.0,
160.0, 55.5, 201.8, 0, 0, 0, 0, 0, 0, 2, 111, 5, 1009, -1, 115, 1,
0, 0, 0, 0, 0, 0, 0, 0, 0, 'No Label'
]]
prep_inputData = pd.DataFrame(inputData, columns=columns)
prediction = mlengine.predict(prep_inputData)
# print(prediction)
self.assertEquals(prediction, [-1])
# if __name__ == '__main__':
# unittest.main()
def __init__(self, remote_ip, remote_port):
self.udp_sender = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket_table = {}
#self.poller = zmq.Poller()
#self.initialize_connections()
self.remote_ip = remote_ip
self.remote_port = remote_port
self.task_sema = threading.Semaphore(0)
self.task_mutex = threading.Lock()
self.predictor = MLEngine(D2LogisticRegression())
self.broadcast_thread = threading.Thread(target=self.start_process, args = ())
self.broadcast_thread.start()
def __init__(self, local_ip, local_port, remote_ip, remote_port):
self.recommender = MLEngine(D2LogisticRegression())
self.local_ip = local_ip
self.local_port = local_port
self.remote_ip = remote_ip
self.remote_port = remote_port
self.heart_beater = HeartBeat("54.186.108.36:8888", remote_ip, remote_port)
self.task_handler = TaskHandler(remote_ip, remote_port)
self.client_thread = threading.Thread(target=self.start_task_creator, args = ())
self.remote_thread = threading.Thread(target=self.start_remote_handler, args = ())
self.heart_beater.start()
self.client_thread.start()
self.remote_thread.start()
self.client_thread.join()
self.remote_thread.join()
self.task_handler.join()
self.heart_beater.stop()
class TaskHandler:
PEER_REPICK = 5
def __init__(self, remote_ip, remote_port):
self.udp_sender = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket_table = {}
#self.poller = zmq.Poller()
#self.initialize_connections()
self.remote_ip = remote_ip
self.remote_port = remote_port
self.task_sema = threading.Semaphore(0)
self.task_mutex = threading.Lock()
self.predictor = MLEngine(D2LogisticRegression())
self.broadcast_thread = threading.Thread(target=self.start_process, args = ())
self.broadcast_thread.start()
def initialize_connections(self):
if len(self.socket_table) < TaskHandler.PEER_REPICK:
if DBGMODE : print("initialize_connections [start]")
DataStore.mutex.acquire()
self.datastore = DataStore()
self.peers = self.datastore.get_highest_rating(10)
for peer in self.peers:
if peer.id not in self.socket_table and (peer.ip != self.remote_ip or peer.port !=self.remote_port):
self.socket_table[peer.id] = (peer.ip, int(peer.port))
self.datastore.close()
DataStore.mutex.release()
# def initialize_connections(self):
# if len(self.socket_table) < TaskHandler.PEER_REPICK:
# if DBGMODE : print("initialize_connections [start]")
# DataStore.mutex.acquire()
# self.datastore = DataStore()
# self.peers = self.datastore.get_highest_rating(2)
# context = zmq.Context()
# for peer in self.peers:
# if peer.id not in self.socket_table:
# #print("connect to: tcp://{}".format(peer))
# socket_item = context.socket(zmq.REQ)
# socket_item.connect ("tcp://{}:{}".format(peer.ip, peer.port))
# self.poller.register(socket_item, zmq.POLLIN)
# self.socket_table[peer.id] = socket_item
# self.datastore.close()
# DataStore.mutex.release()
# print("initialize_connections [end]")
def join(self):
self.broadcast_thread.join()
def launch_task(self, task):
if DBGMODE : print("launch_task [start]")
self.current_task = task
self.task_sema.release()
if task.camp == 0:
local_result = self.predictor.recommend(task.radiant, task.dire)
else:
local_result = self.predictor.recommend(task.dire, task.radiant)
self.current_task.consume(local_result)
self.check_and_done()
if DBGMODE : print("launch_task [end]")
def check_and_done(self):
if DBGMODE : print("check_and_done [start]")
self.task_mutex.acquire()
if self.current_task != None and self.current_task.finished():
#recommendations = self.recommender.recommend(command["radiant"], command["dire"]);
result = self.process_responses(self.current_task.result)
json_str = json.dumps(result);
self.current_task.socket.sendto(json_str.encode("utf-8"), self.current_task.addr)
print("response message:"+json_str)
self.current_task = None
self.task_mutex.release()
if DBGMODE : print("check_and_done [end]")
def process_responses(self,responses):
if DBGMODE : print("process_responses [start]")
result = {}
result["candidates"] = []
print("process_response: \t"+json.dumps(responses))
for i in range(len(responses)):
result["candidates"].append({"heroId": responses[i][1], "rate":responses[i][0]})
if DBGMODE : print("process_responses [end]")
return result
def start_process(self):
while True:
self.initialize_connections()
if DBGMODE : print("pre start process task.")
self.task_sema.acquire()
if DBGMODE : print("real process task.")
for pid in self.socket_table:
target = self.socket_table[pid]
message = json.dumps(self.current_task.request)
print(target)
self.udp_sender.sendto(message.encode("utf-8"), target)
result = []
active_entry = []
remain_time = RETRY_TIMES*TIME_BOUND
start_time = time.time()
count = 0;
print("remain tiem is:"+str(remain_time))
#for count in range(RETRY_TIMES):
while True:
try:
print("[start] recvfrom remote")
self.udp_sender.settimeout(remain_time)
data,addr = self.udp_sender.recvfrom(4096)
message = data.decode("utf-8")
print("[middle] recvfrom:"+message)
element = json.loads(message)
active_entry.append({"ip":element["ip"], "port":element["port"]})
result.extend(element["candidates"])
count += 1;
remain_time = remain_time + start_time - time.time()
#print("[end] recvfrom remote: counter")
#print("len = "+str(len(self.socket_table)))
#print("count = "+str(count))
#print("remain_time = "+str(remain_time))
if len(self.socket_table) <= count or remain_time <= 0:
break
except:
break
dead_list = []
for peer in self.peers:
active = False
for entry in active_entry:
if peer.ip == entry["ip"] and peer.port == entry["port"]:
active = True
if not active:
dead_list.append(peer.id)
for pid in dead_list:
#self.poller.unregister(self.socket_table[pid])
print("close pid: "+str(pid))
#self.socket_table[pid].close()
self.socket_table.pop(pid, None)
self.peers = [peer for peer in self.peers if peer.id not in dead_list]
# delete dead peers[end]
print("peers size ", len(self.peers))
print("remote resulting message {}".format(json.dumps(result)))
self.current_task.consume(result)
self.check_and_done()
