def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client Returns: inserted_id """ event_data.machine_name = \ network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short(event_data.ip)) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format(event_data.ip, event_data.module_name, event_data.machine_name, event_data.data)) return events_data.insert_one(event_data.__dict__).inserted_id
def insert_honeypot_events_data_from_module_processor(ip, module_name, date, data): """ insert data which is received from honeypot modules args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( ip, module_name, network_configuration()["real_machine_identifier_name"], data)) return honeypot_events_data.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "data": data, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: event_data: contain ip, module_name, machine_name, date, data Returns: inserted_id """ event_data.machine_name = network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short( event_data.ip ) ) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( event_data.ip, event_data.module_name, event_data.machine_name, event_data.data ) ) return data_events.insert_one(event_data.__dict__).inserted_id
def insert_pcap_files_to_collection(file_archive: FileArchive): """ Insert the pcap files containing the captured network traffic to mongodb collection Args: file_archive: path of the file Returns: file_id """ if is_verbose_mode(): verbose_info( "Received network traffic file:{0}, date:{1}. " "Inserting it in the File Archive".format( file_archive.file_path, file_archive.date ) ) return ohp_file_archive_gridfs.put( open(file_archive.file_path, "rb"), filename=os.path.split(file_archive.file_path)[1], machine_name=network_configuration()["real_machine_identifier_name"], date=file_archive.date, splitTimeout=file_archive.split_timeout )
def insert_to_credential_events_collection(credential_event: CredentialEvent): """ insert credentials from honeypot events which are obtained from the module processor to credential_event collection Args: credential_event: Object of CredentialEvent Class with honeypot event credentials Returns: inserted_id """ credential_event.country = byte_to_str( IP2Location.get_country_short( credential_event.ip ) ) credential_event.machine_name = network_config["real_machine_identifier_name"] if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( credential_event.ip, credential_event.username, credential_event.password, credential_event.module_name, credential_event.machine_name ) ) return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_to_file_change_events_collection(file_change_event_data: FileEventsData): """ insert file change events which are obtained from ftp/ssh weak_password module Args: file_change_event_data: Object of FileEventsData Class with file change parameters Returns: inserted_id """ file_change_event_data.machine_name = network_config["real_machine_identifier_name"] file_change_event_data.file_content = open( file_change_event_data.file_path, 'rb' ).read() if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else "" if is_verbose_mode(): verbose_info( "Received honeypot file change event, file_path:{0}, status:{1}, " "module_name:{2}, module_name:{3}, machine_name:{3}".format( file_change_event_data.file_path, file_change_event_data.status, file_change_event_data.module_name, file_change_event_data.machine_name, ) ) return file_change_events.insert_one(file_change_event_data.__dict__).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent): """ insert other network events (port scan, etc..) to network_events collection Args: network_event: Object of NetworkEvent Class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( network_event.ip_dest, network_event.port_dest, network_event.ip_src, network_event.port_src, network_event.machine_name)) # Get country of the source IP Address network_event.country_ip_src = byte_to_str( IP2Location.get_country_short(network_event.ip_src)) # Get country of the destination IP Address network_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(network_event.ip_dest)) network_events_queue.append(network_event.__dict__) return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent): """ insert selected modules event to honeypot_events collection Args: honeypot_event: Object of HoneypotEvent class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(honeypot_event.ip_dest, honeypot_event.port_dest, honeypot_event.ip_src, honeypot_event.port_src, honeypot_event.module_name, honeypot_event.machine_name)) # Get country of the source IP Address honeypot_event.country_ip_src = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_src)) # Get country of the destination IP Address honeypot_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_dest)) honeypot_events_queue.append(honeypot_event.__dict__) return
def insert_honeypot_events_credential_from_module_processor( ip, username, password, module_name, date): """ insert honeypot events which are obtained from the modules args: ip : client ip used for connecting to the module username : username tried for connecting to modules password : password tried for connecting to modules module_name : on which module client accessed date : datetime of the event :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( ip, username, password, module_name, network_configuration()["real_machine_identifier_name"])) return credential_events.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "username": username, "password": password, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def create_new_images(configuration): """ start new images based on configuration and dockerfile Args: configuration: user final configuration Returns: True """ for selected_module in configuration: # go to tmp folder to create Dockerfile and files dir tmp_dir_name = make_tmp_thread_dir() os.chdir(tmp_dir_name) # create files dir mkdir("files") # create Dockerfile dockerfile = open("Dockerfile", "w") dockerfile.write(configuration[selected_module]["dockerfile"]) dockerfile.close() # copy files copy_dir_tree(configuration[selected_module]["files"], "files") # create docker image image_name = virtual_machine_name_to_container_name( configuration[selected_module]["virtual_machine_name"], selected_module ) info("creating image {0}".format(image_name)) # in case if verbose mode is enabled, we will be use os.system # instead of os.popen to show the outputs in case # of anyone want to be aware what's happening or what's the error, # it's a good feature for developers as well # to create new modules if is_verbose_mode(): os.system("docker build . -t {0}".format(image_name)) else: os.popen("docker build . -t {0}".format(image_name)).read() # created info("image {0} created".format(image_name)) # go back to home directory os.chdir("../..") # submit tmp dir name tmp_directories.append(tmp_dir_name) return True
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src, module_name, machine_name): """ insert selected modules event to honeypot_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port module_name: module name ran on the port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(ip_dest, port_dest, ip_src, port_src, module_name, machine_name)) global honeypot_events_queue honeypot_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "module_name": module_name, "date": now(), "machine_name": machine_name, "event_type": "honeypot_event", "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def insert_events_in_bulk(): """ inserts all honeypot and network events in bulk to honeypot_events and network_events collection respectively """ global honeypot_events_queue global network_events_queue if is_verbose_mode() and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") if honeypot_events_queue: new_events = honeypot_events_queue[:] honeypot_events_queue = [] honeypot_events.insert_many(new_events) if network_events_queue: new_events = network_events_queue[:] network_events_queue = [] network_events.insert_many(new_events) return
def verbose_info(content): """ build the info message, log the message in database if requested, rewrite the thread temporary file Args: content: content of the message Returns: None """ if is_verbose_mode(): logger.info(content) sys.stdout.buffer.write( bytes( color.color_cmd("cyan") + "[v] [{0}] ".format(now()) + color.color_cmd("grey") + content + color.color_cmd("reset") + "\n", "utf8")) sys.stdout.flush() return
def push_events_queues_to_database(): """ Pushes all honeypot and network events collected in the honeypot_events_queue and network_events_queue to honeypot_events and network_events collection respectively """ if is_verbose_mode() and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") # Insert all honeypot events to database (honeypot_events collection) if honeypot_events_queue: new_events = honeypot_events_queue[:] honeypot_events_queue.clear() honeypot_events.insert_many(new_events) # Insert all network events to database (network_events collection) if network_events_queue: new_events = network_events_queue[:] network_events_queue.clear() network_events.insert_many(new_events) return
def push_events_queues_to_database(honeypot_events_queue, network_events_queue): """ Pushes all honeypot and network events collected in the honeypot_events_queue and network_events_queue to honeypot_events and network_events collection respectively """ if is_verbose_mode() and (honeypot_events_queue or network_events_queue) \ and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") # Insert all honeypot events to database (honeypot_events collection) while not honeypot_events_queue.empty(): new_event = honeypot_events_queue.get() honeypot_events.insert_one(new_event) # Insert all network events to database (network_events collection) while not network_events_queue.empty(): new_event = network_events_queue.get() network_events.insert_one(new_event) return
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src, machine_name): """ insert other network events (port scan, etc..) to network_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( ip_dest, port_dest, ip_src, port_src, machine_name)) global network_events_queue network_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "date": now(), "machine_name": machine_name, "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def test_run_with_verbose_option(self): testargs = ['python3', 'ohp.py', '--verbose'] with patch.object(sys, 'argv', testargs): returned_value = is_verbose_mode() self.assertEqual(returned_value, True)
def network_traffic_capture(configuration, honeypot_events_queue, network_events_queue, network_config): """ Capture network traffic and submit new network and honeypot events to the database Args: configuration: user final configuration honeypot_events_queue: multiprocessing queue for storing honeypot events network_events_queue: multiprocessing queue for storing network events network_config: network configuration Returns: True """ info(messages["network_traffic_capture_start"]) for selected_module in configuration: port_number = configuration[selected_module][ "real_machine_port_number"] honeypot_ports[port_number] = selected_module # get ip addresses virtual_machine_ip_addresses = [ configuration[selected_module]["ip_address"] for selected_module in configuration ] # Ignore VM IPs + IPs in config.py # VM = virtual machine, RM = real machine ignore_rm_ip_addresses = network_config["ignore_real_machine_ip_address"] ignore_vm_ip_addresses = network_config[ "ignore_virtual_machine_ip_addresses"] # Ignore real machine IPs ignore_ip_addresses = network_config["ignore_real_machine_ip_addresses"] \ if ignore_rm_ip_addresses else [] + virtual_machine_ip_addresses \ if ignore_vm_ip_addresses else [] ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration)) # Ignore ports ignore_ports = network_config["ignore_real_machine_ports"] # Display filter to be applied to the Live Captured network traffic display_filter = ' and '.join( ['ip.src!={0} and ip.dst!={0}'.format(_) for _ in ignore_ip_addresses]) display_filter += ' and ' if ignore_ip_addresses and ignore_ports else "" display_filter += ' and '.join([ 'tcp.srcport!={0} and tcp.dstport!={0}'.format(_) for _ in ignore_ports ]) store_pcap = network_config["store_network_captured_files"] timeout = network_config["split_pcap_file_timeout"] # Make the pcapfiles directory for storing the Network captured files base_dir_path = os.path.join(sys.path[0], "pcapfiles") def packet_callback(packet): """ Callback function, called by apply_on_packets Args: packet """ process_packet(packet, honeypot_events_queue, network_events_queue) # Run infinite loop and split the capture in multiple files using the timeout set # in the network configuration while True: # Timestamp to be used in file name file_timestamp = int(time.time()) generation_time = datetime.fromtimestamp(file_timestamp).strftime( "%Y-%m-%d %H:%M:%S") # File path of the network capture file with the timestamp output_file_path = os.path.join( base_dir_path, "captured-traffic-" + str(file_timestamp) + ".pcap") if store_pcap: info(messages["network_capture_getting_stored"].format( output_file_path)) try: capture = pyshark.LiveCapture( interface='any', display_filter=display_filter, output_file=output_file_path if store_pcap else None) # Debug option for pyshark capture if is_verbose_mode(): capture.set_debug() # Applied on every packet captured by pyshark LiveCapture capture.apply_on_packets(packet_callback, timeout=timeout) except get_timeout_error() as e: force_kill_tshark() # Catches the timeout error thrown by apply_on_packets insert_pcap_files_to_collection( FileArchive(output_file_path, generation_time, timeout)) if store_pcap else e except KeyboardInterrupt as e: force_kill_tshark() insert_pcap_files_to_collection( FileArchive(output_file_path, generation_time, timeout)) if store_pcap else e break except Exception as e: force_kill_tshark() insert_pcap_files_to_collection( FileArchive(output_file_path, generation_time, timeout)) if store_pcap else e error(str(e)) break return True