def insert_to_network_events_queue(network_event: NetworkEvent): """ insert other network events (port scan, etc..) to network_events collection Args: network_event: Object of NetworkEvent Class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( network_event.ip_dest, network_event.port_dest, network_event.ip_src, network_event.port_src, network_event.machine_name)) # Get country of the source IP Address network_event.country_ip_src = byte_to_str( IP2Location.get_country_short(network_event.ip_src)) # Get country of the destination IP Address network_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(network_event.ip_dest)) network_events_queue.append(network_event.__dict__) return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent): """ insert selected modules event to honeypot_events collection Args: honeypot_event: Object of HoneypotEvent class with event parameters Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(honeypot_event.ip_dest, honeypot_event.port_dest, honeypot_event.ip_src, honeypot_event.port_src, honeypot_event.module_name, honeypot_event.machine_name)) # Get country of the source IP Address honeypot_event.country_ip_src = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_src)) # Get country of the destination IP Address honeypot_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_dest)) honeypot_events_queue.append(honeypot_event.__dict__) return
def insert_pcap_files_to_collection(file_archive: FileArchive): """ Insert the pcap files containing the captured network traffic to mongodb collection Args: file_archive: path of the file Returns: file_id """ if is_verbose_mode(): verbose_info( "Received network traffic file:{0}, date:{1}. " "Inserting it in the File Archive".format( file_archive.file_path, file_archive.date ) ) return ohp_file_archive_gridfs.put( open(file_archive.file_path, "rb"), filename=os.path.split(file_archive.file_path)[1], machine_name=network_configuration()["real_machine_identifier_name"], date=file_archive.date, splitTimeout=file_archive.split_timeout )
def insert_honeypot_events_credential_from_module_processor( ip, username, password, module_name, date): """ insert honeypot events which are obtained from the modules args: ip : client ip used for connecting to the module username : username tried for connecting to modules password : password tried for connecting to modules module_name : on which module client accessed date : datetime of the event :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( ip, username, password, module_name, network_configuration()["real_machine_identifier_name"])) return credential_events.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "username": username, "password": password, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent, network_events_queue: Queue): """ insert other network events (port scan, etc..) to network_events_queue Args: network_event: Object of NetworkEvent Class with event parameters network_events_queue: Multiprocessing queue which stores the list of network_events in _dict_ format Returns: None """ verbose_info(messages["received_network_event"].format( network_event.ip_dest, network_event.port_dest, network_event.ip_src, network_event.port_src, network_event.machine_name)) # Get country of the source IP Address network_event.country_ip_src = byte_to_str( IP2Location.get_country_short(network_event.ip_src)) # Get country of the destination IP Address network_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(network_event.ip_dest)) network_events_queue.put(network_event.__dict__) return
def insert_to_file_change_events_collection( file_change_event_data: FileEventsData): """ insert file change events which are obtained from ftp/ssh weak_password module Args: file_change_event_data: Object of FileEventsData Class with file change parameters Returns: inserted_id """ file_change_event_data.machine_name = network_config[ "real_machine_identifier_name"] file_change_event_data.file_content = binascii.b2a_base64( open(file_change_event_data.file_path, 'rb').read() ).decode( ) if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else "" verbose_info(messages["received_honeypot_file_change_event"].format( file_change_event_data.file_path, file_change_event_data.status, file_change_event_data.module_name, file_change_event_data.machine_name, )) return elasticsearch_events.index(index='file_change_events', body=file_change_event_data.__dict__)
def insert_pcap_files_to_collection(file_archive: FileArchive): """ Insert the pcap files containing the captured network traffic to mongodb collection Args: file_archive: path of the file Returns: file_id """ verbose_info(messages["received_network_traffic_file"].format( file_archive.file_path, file_archive.date)) file_content = binascii.b2a_base64( open(file_archive.file_path, "rb").read()).decode() file_md5 = hashlib.md5(file_content.encode()).hexdigest() return elasticsearch_events.index( index='ohp_file_archive', body={ "md5": file_md5, "content": file_content, "filename": os.path.split(file_archive.file_path)[1], "machine_name": network_configuration()["real_machine_identifier_name"], "date": file_archive.date, "splitTimeout": file_archive.split_timeout })
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent, honeypot_events_queue: Queue): """ insert selected modules event to honeypot_events_queue Args: honeypot_event: Object of HoneypotEvent class with event parameters honeypot_events_queue: Multiprocessing queue which stores the list of honeypot_events in _dict_ format Returns: None """ verbose_info(messages["received_honeypot_event"].format( honeypot_event.ip_dest, honeypot_event.port_dest, honeypot_event.ip_src, honeypot_event.port_src, honeypot_event.module_name, honeypot_event.machine_name)) # Get country of the source IP Address honeypot_event.country_ip_src = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_src)) # Get country of the destination IP Address honeypot_event.country_ip_dest = byte_to_str( IP2Location.get_country_short(honeypot_event.ip_dest)) honeypot_events_queue.put(honeypot_event.__dict__) return
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client Returns: inserted_id """ event_data.machine_name = \ network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short(event_data.ip)) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format(event_data.ip, event_data.module_name, event_data.machine_name, event_data.data)) return events_data.insert_one(event_data.__dict__).inserted_id
def push_events_queues_to_database(honeypot_events_queue, network_events_queue): """ Pushes all honeypot and network events collected in the honeypot_events_queue and network_events_queue to honeypot_events and network_events collection respectively Args: honeypot_events_queue: Multiprocessing queue which stores the list of honeypot_events in _dict_ format network_events_queue: Multiprocessing queue which stores the list of network_events in _dict_ format """ if honeypot_events_queue or network_events_queue: verbose_info(messages["submitting_events"]) # Insert all honeypot events to database (honeypot_events collection) while not honeypot_events_queue.empty(): new_event = honeypot_events_queue.get() elasticsearch_events.index(index='honeypot_events', body=new_event) # Insert all network events to database (network_events collection) while not network_events_queue.empty(): new_event = network_events_queue.get() elasticsearch_events.index(index='network_events', body=new_event) return
def insert_honeypot_events_data_from_module_processor(ip, module_name, date, data): """ insert data which is received from honeypot modules args: ip : client ip used for putting the data module_name : on which module client accessed date : datetime of the events data : Data which is obtained from the client :return: inserted_id """ if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( ip, module_name, network_configuration()["real_machine_identifier_name"], data)) return honeypot_events_data.insert_one({ "ip_dest": byte_to_str(ip), "module_name": module_name, "date": date, "data": data, "country": byte_to_str(IP2Location.get_country_short(byte_to_str(ip))), "machine_name": network_configuration()["real_machine_identifier_name"] }).inserted_id
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: event_data: contain ip, module_name, machine_name, date, data Returns: inserted_id """ event_data.machine_name = network_config["real_machine_identifier_name"] event_data.country = byte_to_str( IP2Location.get_country_short( event_data.ip ) ) if is_verbose_mode(): verbose_info( "Received honeypot data event, ip_dest:{0}, module_name:{1}, " "machine_name:{2}, data:{3}".format( event_data.ip, event_data.module_name, event_data.machine_name, event_data.data ) ) return data_events.insert_one(event_data.__dict__).inserted_id
def insert_to_file_change_events_collection(file_change_event_data: FileEventsData): """ insert file change events which are obtained from ftp/ssh weak_password module Args: file_change_event_data: Object of FileEventsData Class with file change parameters Returns: inserted_id """ file_change_event_data.machine_name = network_config["real_machine_identifier_name"] file_change_event_data.file_content = open( file_change_event_data.file_path, 'rb' ).read() if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else "" if is_verbose_mode(): verbose_info( "Received honeypot file change event, file_path:{0}, status:{1}, " "module_name:{2}, module_name:{3}, machine_name:{3}".format( file_change_event_data.file_path, file_change_event_data.status, file_change_event_data.module_name, file_change_event_data.machine_name, ) ) return file_change_events.insert_one(file_change_event_data.__dict__).inserted_id
def insert_to_credential_events_collection(credential_event: CredentialEvent): """ insert credentials from honeypot events which are obtained from the module processor to credential_event collection Args: credential_event: Object of CredentialEvent Class with honeypot event credentials Returns: inserted_id """ credential_event.country = byte_to_str( IP2Location.get_country_short( credential_event.ip ) ) credential_event.machine_name = network_config["real_machine_identifier_name"] if is_verbose_mode(): verbose_info( "Received honeypot credential event, ip_dest:{0}, username:{1}, " "password:{2}, module_name:{3}, machine_name:{4}".format( credential_event.ip, credential_event.username, credential_event.password, credential_event.module_name, credential_event.machine_name ) ) return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src, module_name, machine_name): """ insert selected modules event to honeypot_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port module_name: module name ran on the port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info( "Received honeypot event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}". format(ip_dest, port_dest, ip_src, port_src, module_name, machine_name)) global honeypot_events_queue honeypot_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "module_name": module_name, "date": now(), "machine_name": machine_name, "event_type": "honeypot_event", "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def insert_events_in_bulk(): """ inserts all honeypot and network events in bulk to honeypot_events and network_events collection respectively """ global honeypot_events_queue global network_events_queue if is_verbose_mode() and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") if honeypot_events_queue: new_events = honeypot_events_queue[:] honeypot_events_queue = [] honeypot_events.insert_many(new_events) if network_events_queue: new_events = network_events_queue[:] network_events_queue = [] network_events.insert_many(new_events) return
def insert_to_events_data_collection(event_data: EventData): """ Insert data collected from module processors of modules such as- ICS module Args: event_data: contain ip, module_name, machine_name, date, data Returns: inserted_id """ event_data.machine_name = network_config["real_machine_identifier_name"] event_data.country_ip_src = byte_to_str( IP2Location.get_country_short(event_data.ip_src)) verbose_info(messages["received_honeypot_data_event"].format( event_data.ip_src, event_data.module_name, event_data.machine_name, event_data.data)) return elasticsearch_events.index(index='data_events', body=event_data.__dict__)
def submit_report_to_db(event): """ this function created to submit the generated reports into db, the files are not stored in db, just the path! Args: event: event log Returns: return True if submitted otherwise False """ verbose_info(messages("inserting_report_db")) session = create_connection() session.add( Report( date=event["date"], scan_unique_id=event["scan_unique_id"], report_path_filename=json.dumps( event["options"]["report_path_filename"]), options=json.dumps(event["options"]), )) return send_submit_query(session)
def push_events_queues_to_database(): """ Pushes all honeypot and network events collected in the honeypot_events_queue and network_events_queue to honeypot_events and network_events collection respectively """ if is_verbose_mode() and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") # Insert all honeypot events to database (honeypot_events collection) if honeypot_events_queue: new_events = honeypot_events_queue[:] honeypot_events_queue.clear() honeypot_events.insert_many(new_events) # Insert all network events to database (network_events collection) if network_events_queue: new_events = network_events_queue[:] network_events_queue.clear() network_events.insert_many(new_events) return
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src, machine_name): """ insert other network events (port scan, etc..) to network_events collection Args: ip_dest: dest ip (machine) port_dest: dest port (machine) ip_src: src ip port_src: src port machine_name: real machine name Returns: ObjectId(inserted_id) """ if is_verbose_mode(): verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, " "ip_src:{2}, port_src:{3}, machine_name:{4}".format( ip_dest, port_dest, ip_src, port_src, machine_name)) global network_events_queue network_events_queue.append({ "ip_dest": byte_to_str(ip_dest), "port_dest": int(port_dest), "ip_src": byte_to_str(ip_src), "port_src": int(port_src), "date": now(), "machine_name": machine_name, "country_ip_src": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))), "country_ip_dest": byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest))) }) return
def push_events_queues_to_database(honeypot_events_queue, network_events_queue): """ Pushes all honeypot and network events collected in the honeypot_events_queue and network_events_queue to honeypot_events and network_events collection respectively """ if is_verbose_mode() and (honeypot_events_queue or network_events_queue) \ and (honeypot_events_queue or network_events_queue): verbose_info("Submitting new events to database") # Insert all honeypot events to database (honeypot_events collection) while not honeypot_events_queue.empty(): new_event = honeypot_events_queue.get() honeypot_events.insert_one(new_event) # Insert all network events to database (network_events collection) while not network_events_queue.empty(): new_event = network_events_queue.get() network_events.insert_one(new_event) return
def insert_to_credential_events_collection(credential_event: CredentialEvent): """ insert credentials from honeypot events which are obtained from the module processor to credential_event collection Args: credential_event: Object of CredentialEvent Class with honeypot event credentials Returns: inserted_id """ credential_event.country_ip_src = byte_to_str( IP2Location.get_country_short(credential_event.ip_src)) credential_event.machine_name = network_config[ "real_machine_identifier_name"] verbose_info(messages["received_honeypot_credential_event"].format( credential_event.ip_src, credential_event.username, credential_event.password, credential_event.module_name, credential_event.machine_name)) return elasticsearch_events.index(index='credential_events', body=credential_event.__dict__)
def process_conditions( event, module_name, target, scan_unique_id, options, response, process_number, module_thread_number, total_module_thread_number, request_number_counter, total_number_of_requests ): from core.alert import (success_event_info, verbose_info, messages) if 'save_to_temp_events_only' in event.get('response', ''): from database.db import submit_temp_logs_to_db submit_temp_logs_to_db( { "date": now(model=None), "target": target, "module_name": module_name, "scan_unique_id": scan_unique_id, "event_name": event['response']['save_to_temp_events_only'], # "options": options, "options": {}, "event": event, "data": response } ) if event['response']['conditions_results'] and 'save_to_temp_events_only' not in event.get('response', ''): from database.db import submit_logs_to_db # remove sensitive information before submitting to db from config import nettacker_api_config options = copy.deepcopy(options) for key in nettacker_api_config(): try: del options[key] except Exception: continue del event['response']['conditions'] event_request_keys = copy.deepcopy(event) del event_request_keys['response'] submit_logs_to_db( { "date": now(model=None), "target": target, "module_name": module_name, "scan_unique_id": scan_unique_id, # "options": options, "options": {}, "event": event } ) success_event_info( messages("send_success_event_from_module").format( process_number, module_name, target, module_thread_number, total_module_thread_number, request_number_counter, total_number_of_requests, ", ".join( [ "{}: {}".format( key, event_request_keys[key] ) for key in event_request_keys ] ), ", ".join(event['response']['conditions_results'].keys()) ) ) verbose_info( json.dumps(event) ) return True else: del event['response']['conditions'] verbose_info( messages("send_unsuccess_event_from_module").format( process_number, module_name, target, module_thread_number, total_module_thread_number, request_number_counter, total_number_of_requests ) ) verbose_info( json.dumps(event) ) return 'save_to_temp_events_only' in event['response']