def insert_honeypot_events_data_from_module_processor(ip, module_name, date, data): """ insert data which is received from honeypot modules args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( ip, module_name, network_configuration()["real_machine_identifier_name"], data)) return honeypot_events_data.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "data": data, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent, honeypot_events_queue: Queue): """ insert selected modules event to honeypot_events_queue Args: honeypot_event: Object of HoneypotEvent class with event parameters honeypot_events_queue: Multiprocessing queue which stores the list of honeypot_events in _dict_ format Returns: None """ verbose_info(messages["received_honeypot_event"].format( honeypot_event.ip_dest, honeypot_event.port_dest, honeypot_event.ip_src, honeypot_event.port_src, honeypot_event.module_name, honeypot_event.machine_name)) # Get country of the source IP Address honeypot_event.country_ip_src = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_src)) # Get country of the destination IP Address honeypot_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_dest)) honeypot_events_queue.put(honeypot_event.__dict__) return
def insert_honeypot_events_credential_from_module_processor( ip, username, password, module_name, date): """ insert honeypot events which are obtained from the modules args: ip : client ip used for connecting to the module username : username tried for connecting to modules password : password tried for connecting to modules module_name : on which module client accessed date : datetime of the event :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( ip, username, password, module_name, network_configuration()["real_machine_identifier_name"])) return credential_events.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "username": username, "password": password, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent, network_events_queue: Queue): """ insert other network events (port scan, etc..) to network_events_queue Args: network_event: Object of NetworkEvent Class with event parameters network_events_queue: Multiprocessing queue which stores the list of network_events in _dict_ format Returns: None """ verbose_info(messages["received_network_event"].format( network_event.ip_dest, network_event.port_dest, network_event.ip_src, network_event.port_src, network_event.machine_name)) # Get country of the source IP Address network_event.country_ip_src = byte_to_str( IP2Location.get_country_short(network_event.ip_src)) # Get country of the destination IP Address network_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(network_event.ip_dest)) network_events_queue.put(network_event.__dict__) return
def insert_to_network_events_queue(network_event: NetworkEvent): """ insert other network events (port scan, etc..) to network_events collection Args: network_event: Object of NetworkEvent Class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( network_event.ip_dest, network_event.port_dest, network_event.ip_src, network_event.port_src, network_event.machine_name)) # Get country of the source IP Address network_event.country_ip_src = byte_to_str( IP2Location.get_country_short(network_event.ip_src)) # Get country of the destination IP Address network_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(network_event.ip_dest)) network_events_queue.append(network_event.__dict__) return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent): """ insert selected modules event to honeypot_events collection Args: honeypot_event: Object of HoneypotEvent class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(honeypot_event.ip_dest, honeypot_event.port_dest, honeypot_event.ip_src, honeypot_event.port_src, honeypot_event.module_name, honeypot_event.machine_name)) # Get country of the source IP Address honeypot_event.country_ip_src = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_src)) # Get country of the destination IP Address honeypot_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_dest)) honeypot_events_queue.append(honeypot_event.__dict__) return
def on_any_event(self, event): if not (event.event_type == 'modified' and event.is_directory) \ and not is_excluded(event.src_path, self.EXCLUDES): insert_to_file_change_events_collection( FileEventsData(file_path=byte_to_str(event.src_path), status=byte_to_str(event.event_type), module_name=self.module_name, date=now(), is_directory=event.is_directory))
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: event_data: contain ip, module_name, machine_name, date, data Returns: inserted_id """ event_data.machine_name = network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short( event_data.ip ) ) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( event_data.ip, event_data.module_name, event_data.machine_name, event_data.data ) ) return data_events.insert_one(event_data.__dict__).inserted_id
def insert_to_credential_events_collection(credential_event: CredentialEvent): """ insert credentials from honeypot events which are obtained from the module processor to credential_event collection Args: credential_event: Object of CredentialEvent Class with honeypot event credentials Returns: inserted_id """ credential_event.country = byte_to_str( IP2Location.get_country_short( credential_event.ip ) ) credential_event.machine_name = network_config["real_machine_identifier_name"] if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( credential_event.ip, credential_event.username, credential_event.password, credential_event.module_name, credential_event.machine_name ) ) return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client Returns: inserted_id """ event_data.machine_name = \ network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short(event_data.ip)) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format(event_data.ip, event_data.module_name, event_data.machine_name, event_data.data)) return events_data.insert_one(event_data.__dict__).inserted_id
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: event_data: contain ip, module_name, machine_name, date, data Returns: inserted_id """ event_data.machine_name = network_config["real_machine_identifier_name"] event_data.country_ip_src = byte_to_str( IP2Location.get_country_short(event_data.ip_src)) verbose_info(messages["received_honeypot_data_event"].format( event_data.ip_src, event_data.module_name, event_data.machine_name, event_data.data)) return elasticsearch_events.index(index='data_events', body=event_data.__dict__)
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src, module_name, machine_name): """ insert selected modules event to honeypot_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port module_name: module name ran on the port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(ip_dest, port_dest, ip_src, port_src, module_name, machine_name)) global honeypot_events_queue honeypot_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "module_name": module_name, "date": now(), "machine_name": machine_name, "event_type": "honeypot_event", "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def insert_to_credential_events_collection(credential_event: CredentialEvent): """ insert credentials from honeypot events which are obtained from the module processor to credential_event collection Args: credential_event: Object of CredentialEvent Class with honeypot event credentials Returns: inserted_id """ credential_event.country_ip_src = byte_to_str( IP2Location.get_country_short(credential_event.ip_src)) credential_event.machine_name = network_config[ "real_machine_identifier_name"] verbose_info(messages["received_honeypot_credential_event"].format( credential_event.ip_src, credential_event.username, credential_event.password, credential_event.module_name, credential_event.machine_name)) return elasticsearch_events.index(index='credential_events', body=credential_event.__dict__)
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src, machine_name): """ insert other network events (port scan, etc..) to network_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( ip_dest, port_dest, ip_src, port_src, machine_name)) global network_events_queue network_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "date": now(), "machine_name": machine_name, "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def test_byte_to_str(self): content = b"hello" self.assertIsInstance(byte_to_str(content), str)
def new_network_events(configuration): """ get and submit new network events Args: configuration: user final configuration Returns: True """ info("new_network_events thread started") # honeypot ports honeypot_ports = [] for selected_module in configuration: honeypot_ports.append( configuration[selected_module]["real_machine_port_number"]) # set machine name machine_name = network_configuration()["real_machine_identifier_name"] # get ip addresses virtual_machine_ip_addresses = [ configuration[selected_module]["ip_address"] for selected_module in configuration ] # ignore vm ips + ips in config.py ignore_ip_addresses = network_configuration()["ignore_real_machine_ip_addresses"] \ if network_configuration()["ignore_real_machine_ip_address"] else [] + virtual_machine_ip_addresses \ if network_configuration()["ignore_virtual_machine_ip_addresses"] else [] ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration)) # ignore ports ignore_ports = network_configuration()["ignore_real_machine_ports"] # start tshark to capture network # tshark -Y "ip.dst != 192.168.1.1" -T fields -e ip.dst -e tcp.srcport run_tshark = ["tshark", "-l", "-V"] run_tshark.extend(ignore_ip_addresses_rule_generator(ignore_ip_addresses)) run_tshark.extend([ "-T", "fields", "-e", "ip.dst", "-e", "ip.src", "-e", "tcp.dstport", "-e", "tcp.srcport", "-ni", "any" ]) process = subprocess.Popen(run_tshark, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # wait 3 seconds if process terminated? time.sleep(3) if process.poll() is not None: exit_failure("tshark couldn't capture network, maybe run as root!") # todo: replace tshark with python port sniffing - e.g https://www.binarytides.com/python-packet-sniffer-code-linux/ # it will be easier to apply filters and analysis packets with python # if it requires to be run as root, please add a uid checker in framework startup # readline timeout bug fix: https://stackoverflow.com/a/10759061 pull_object = select.poll() pull_object.register(process.stdout, select.POLLIN) # while True, read tshark output try: while True: if pull_object.poll(0): line = process.stdout.readline() # check if new IP and Port printed if len(line) > 0: # split the IP and Port try: line = line.rsplit() ip_dest = byte_to_str(line[0]) ip_src = byte_to_str(line[1]) port_dest = int(line[2]) port_src = int(line[3]) if (netaddr.valid_ipv4(ip_dest) or netaddr.valid_ipv6(ip_dest)) \ and ip_dest not in ignore_ip_addresses \ and ip_src not in ignore_ip_addresses \ and port_dest not in ignore_ports \ and port_src not in ignore_ports: # ignored ip addresses and ports in python - fix later # check if the port is in selected module if port_dest in honeypot_ports or port_src in honeypot_ports: if port_dest in honeypot_ports: insert_selected_modules_network_event( ip_dest, port_dest, ip_src, port_src, selected_module, machine_name) else: insert_other_network_event( ip_dest, port_dest, ip_src, port_src, machine_name) except Exception as _: del _ # check if event shows an IP time.sleep(0.001) # todo: is sleep(0.001) fastest/best? it means it could get 1000 packets per second (maximum) from tshark # how could we prevent the DDoS attacks in here and avoid submitting in MongoDB? should we? except Exception as _: del _ return True