class Engine: def __init__(self, local_ip, local_port, remote_ip, remote_port): self.recommender = MLEngine(D2LogisticRegression()) self.local_ip = local_ip self.local_port = local_port self.remote_ip = remote_ip self.remote_port = remote_port self.heart_beater = HeartBeat("54.186.108.36:8888", remote_ip, remote_port) self.task_handler = TaskHandler(remote_ip, remote_port) self.client_thread = threading.Thread(target=self.start_task_creator, args = ()) self.remote_thread = threading.Thread(target=self.start_remote_handler, args = ()) self.heart_beater.start() self.client_thread.start() self.remote_thread.start() self.client_thread.join() self.remote_thread.join() self.task_handler.join() self.heart_beater.stop() def start_task_creator(self): address = (self.local_ip, int(self.local_port)) server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) server_socket.bind(address) while True: print("root receive new task [start]") content, addr = server_socket.recvfrom(2048) command = json.loads(content.decode("utf-8")) radiant = [i for i in command["radiant"] if i != -1 and i!=77] dire = [i for i in command["dire"] if i != -1 and i!=77] command["dire"] = dire command["radiant"] = radiant responses = self.task_handler.launch_task(Task(command, server_socket, addr)) print("root receive new task [end]") #server_socket.sendto("hello c#".encode("utf-8"), addr) def process_request(self, request): if request["camp"] == 0: return self.recommender.recommend(request["radiant"], request["dire"]) else: return self.recommender.recommend(request["dire"], request["radiant"]) def start_remote_handler(self): sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # UDP sock.bind((self.remote_ip, int(self.remote_port))) result = {} while True: data, addr = sock.recvfrom(4096) message = data.decode("utf-8") print("receive remote request: {}",message) request = json.loads(message) result["candidates"] = self.process_request(request) result["ip"] = self.remote_ip result["port"] = self.remote_port sock.sendto(json.dumps(result).encode("utf-8"), addr) print("send remote response: {}",json.dumps(result))
def runIDS(verbose=False): print("Starting IDS...") try: # Create log file if it does not exist. if not os.path.exists(r'logs\idslogs\ids.log'): file = open(os.path.join(r'logs\idslogs', 'ids.log'), 'w') file.close() logging.basicConfig(filename=os.path.join(r'logs\idslogs', 'ids.log'), level=logging.INFO) csvloader = CSVFlowLoader(os.path.join(CSVFILEPATH, csvfilename)) mlengine = MLEngine(MODEL_FILEPATH, DATACLEAN_PIPELINE_FILEPATH, DATAPREP_PIPELINE_FILEPATH) while True: for flowline in csvloader.tailFile(): csValsArray = [list(flowline.split(","))] csValsDF = pd.DataFrame(csValsArray, columns=columns) # Actual detection and printing results out in stdout. if mlengine.predict(csValsDF)[0] == PredictLabel.ANOMALY.value: print("ANOMALY: %s" % (parsePredictionDF(csValsDF))) logging.info("ANOMALY: %s" % (parsePredictionDF(csValsDF))) else: if verbose: print("BENIGN: %s" % (parsePredictionDF(csValsDF))) except KeyboardInterrupt: print("Exiting...") csvloader.destroy()
def test_predict(self): columns = constants.COLUMNS mlengine = MLEngine() # resultSet = [-1,1] # inputData = [[49188,4,2,0,12,0,6,6,6,0,0,0,0,0,14000,3500,4,0,4,4,4,4,0,4,4,0,0,0,0,0,0,0,0,0,40,0,500000,0,6,6,6,0,0,0,0,0,0,1,1,0,0,0,9,6,0,40,0,0,0,0,0,0,2,12,0,0,329,-1,1,20,0,0,0,0,0,0,0,0]] inputData = [[ '192.168.91.225-13.107.4.52-65026-80-6', '192.168.91.225', 65026, '13.107.4.52', 80, 6, '30/09/2019 07:34:49 PM', 19002, 2, 5, 111.0, 1009.0, 111.0, 0.0, 55.5, 78.48885271170677, 1009.0, 0.0, 201.8, 451.2385178594576, 58941.164087990735, 368.3822755499421, 3167.0, 3013.9242857112386, 7587.0, 135.0, 543.0, 543.0, 0.0, 543.0, 543.0, 19002.0, 4750.5, 2930.5069527301926, 8265.0, 1358.0, 0, 0, 0, 0, 40, 124, 105.25207872855488, 263.1301968213872, 0.0, 1009.0, 140.0, 353.2708390375375, 124800.28571428571, 0, 1, 0, 0, 0, 0, 0, 0, 2.0, 160.0, 55.5, 201.8, 0, 0, 0, 0, 0, 0, 2, 111, 5, 1009, -1, 115, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 'No Label' ]] prep_inputData = pd.DataFrame(inputData, columns=columns) prediction = mlengine.predict(prep_inputData) # print(prediction) self.assertEquals(prediction, [-1]) # if __name__ == '__main__': # unittest.main()
def __init__(self, remote_ip, remote_port): self.udp_sender = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket_table = {} #self.poller = zmq.Poller() #self.initialize_connections() self.remote_ip = remote_ip self.remote_port = remote_port self.task_sema = threading.Semaphore(0) self.task_mutex = threading.Lock() self.predictor = MLEngine(D2LogisticRegression()) self.broadcast_thread = threading.Thread(target=self.start_process, args = ()) self.broadcast_thread.start()
def __init__(self, local_ip, local_port, remote_ip, remote_port): self.recommender = MLEngine(D2LogisticRegression()) self.local_ip = local_ip self.local_port = local_port self.remote_ip = remote_ip self.remote_port = remote_port self.heart_beater = HeartBeat("54.186.108.36:8888", remote_ip, remote_port) self.task_handler = TaskHandler(remote_ip, remote_port) self.client_thread = threading.Thread(target=self.start_task_creator, args = ()) self.remote_thread = threading.Thread(target=self.start_remote_handler, args = ()) self.heart_beater.start() self.client_thread.start() self.remote_thread.start() self.client_thread.join() self.remote_thread.join() self.task_handler.join() self.heart_beater.stop()
class TaskHandler: PEER_REPICK = 5 def __init__(self, remote_ip, remote_port): self.udp_sender = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket_table = {} #self.poller = zmq.Poller() #self.initialize_connections() self.remote_ip = remote_ip self.remote_port = remote_port self.task_sema = threading.Semaphore(0) self.task_mutex = threading.Lock() self.predictor = MLEngine(D2LogisticRegression()) self.broadcast_thread = threading.Thread(target=self.start_process, args = ()) self.broadcast_thread.start() def initialize_connections(self): if len(self.socket_table) < TaskHandler.PEER_REPICK: if DBGMODE : print("initialize_connections [start]") DataStore.mutex.acquire() self.datastore = DataStore() self.peers = self.datastore.get_highest_rating(10) for peer in self.peers: if peer.id not in self.socket_table and (peer.ip != self.remote_ip or peer.port !=self.remote_port): self.socket_table[peer.id] = (peer.ip, int(peer.port)) self.datastore.close() DataStore.mutex.release() # def initialize_connections(self): # if len(self.socket_table) < TaskHandler.PEER_REPICK: # if DBGMODE : print("initialize_connections [start]") # DataStore.mutex.acquire() # self.datastore = DataStore() # self.peers = self.datastore.get_highest_rating(2) # context = zmq.Context() # for peer in self.peers: # if peer.id not in self.socket_table: # #print("connect to: tcp://{}".format(peer)) # socket_item = context.socket(zmq.REQ) # socket_item.connect ("tcp://{}:{}".format(peer.ip, peer.port)) # self.poller.register(socket_item, zmq.POLLIN) # self.socket_table[peer.id] = socket_item # self.datastore.close() # DataStore.mutex.release() # print("initialize_connections [end]") def join(self): self.broadcast_thread.join() def launch_task(self, task): if DBGMODE : print("launch_task [start]") self.current_task = task self.task_sema.release() if task.camp == 0: local_result = self.predictor.recommend(task.radiant, task.dire) else: local_result = self.predictor.recommend(task.dire, task.radiant) self.current_task.consume(local_result) self.check_and_done() if DBGMODE : print("launch_task [end]") def check_and_done(self): if DBGMODE : print("check_and_done [start]") self.task_mutex.acquire() if self.current_task != None and self.current_task.finished(): #recommendations = self.recommender.recommend(command["radiant"], command["dire"]); result = self.process_responses(self.current_task.result) json_str = json.dumps(result); self.current_task.socket.sendto(json_str.encode("utf-8"), self.current_task.addr) print("response message:"+json_str) self.current_task = None self.task_mutex.release() if DBGMODE : print("check_and_done [end]") def process_responses(self,responses): if DBGMODE : print("process_responses [start]") result = {} result["candidates"] = [] print("process_response: \t"+json.dumps(responses)) for i in range(len(responses)): result["candidates"].append({"heroId": responses[i][1], "rate":responses[i][0]}) if DBGMODE : print("process_responses [end]") return result def start_process(self): while True: self.initialize_connections() if DBGMODE : print("pre start process task.") self.task_sema.acquire() if DBGMODE : print("real process task.") for pid in self.socket_table: target = self.socket_table[pid] message = json.dumps(self.current_task.request) print(target) self.udp_sender.sendto(message.encode("utf-8"), target) result = [] active_entry = [] remain_time = RETRY_TIMES*TIME_BOUND start_time = time.time() count = 0; print("remain tiem is:"+str(remain_time)) #for count in range(RETRY_TIMES): while True: try: print("[start] recvfrom remote") self.udp_sender.settimeout(remain_time) data,addr = self.udp_sender.recvfrom(4096) message = data.decode("utf-8") print("[middle] recvfrom:"+message) element = json.loads(message) active_entry.append({"ip":element["ip"], "port":element["port"]}) result.extend(element["candidates"]) count += 1; remain_time = remain_time + start_time - time.time() #print("[end] recvfrom remote: counter") #print("len = "+str(len(self.socket_table))) #print("count = "+str(count)) #print("remain_time = "+str(remain_time)) if len(self.socket_table) <= count or remain_time <= 0: break except: break dead_list = [] for peer in self.peers: active = False for entry in active_entry: if peer.ip == entry["ip"] and peer.port == entry["port"]: active = True if not active: dead_list.append(peer.id) for pid in dead_list: #self.poller.unregister(self.socket_table[pid]) print("close pid: "+str(pid)) #self.socket_table[pid].close() self.socket_table.pop(pid, None) self.peers = [peer for peer in self.peers if peer.id not in dead_list] # delete dead peers[end] print("peers size ", len(self.peers)) print("remote resulting message {}".format(json.dumps(result))) self.current_task.consume(result) self.check_and_done()