def insert_to_network_events_queue(network_event: NetworkEvent):
"""
insert other network events (port scan, etc..) to network_events
collection
Args:
network_event: Object of NetworkEvent Class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
network_event.ip_dest, network_event.port_dest,
network_event.ip_src, network_event.port_src,
network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.append(network_event.__dict__)
return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent):
"""
insert selected modules event to honeypot_events collection
Args:
honeypot_event: Object of HoneypotEvent class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(honeypot_event.ip_dest, honeypot_event.port_dest,
honeypot_event.ip_src, honeypot_event.port_src,
honeypot_event.module_name, honeypot_event.machine_name))
# Get country of the source IP Address
honeypot_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_src))
# Get country of the destination IP Address
honeypot_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_dest))
honeypot_events_queue.append(honeypot_event.__dict__)
return
def insert_pcap_files_to_collection(file_archive: FileArchive):
"""
Insert the pcap files containing the captured network traffic to
mongodb collection
Args:
file_archive: path of the file
Returns:
file_id
"""
if is_verbose_mode():
verbose_info(
"Received network traffic file:{0}, date:{1}. "
"Inserting it in the File Archive".format(
file_archive.file_path,
file_archive.date
)
)
return ohp_file_archive_gridfs.put(
open(file_archive.file_path, "rb"),
filename=os.path.split(file_archive.file_path)[1],
machine_name=network_configuration()["real_machine_identifier_name"],
date=file_archive.date,
splitTimeout=file_archive.split_timeout
)
def insert_honeypot_events_credential_from_module_processor(
ip, username, password, module_name, date):
"""
insert honeypot events which are obtained from the modules
args:
ip : client ip used for connecting to the module
username : username tried for connecting to modules
password : password tried for connecting to modules
module_name : on which module client accessed
date : datetime of the event
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
ip, username, password, module_name,
network_configuration()["real_machine_identifier_name"]))
return credential_events.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"username":
username,
"password":
password,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent,
network_events_queue: Queue):
"""
insert other network events (port scan, etc..) to network_events_queue
Args:
network_event: Object of NetworkEvent Class with event parameters
network_events_queue: Multiprocessing queue which stores the list of
network_events in _dict_ format
Returns:
None
"""
verbose_info(messages["received_network_event"].format(
network_event.ip_dest, network_event.port_dest, network_event.ip_src,
network_event.port_src, network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.put(network_event.__dict__)
return
def insert_to_file_change_events_collection(
file_change_event_data: FileEventsData):
"""
insert file change events which are obtained from ftp/ssh weak_password
module
Args:
file_change_event_data: Object of FileEventsData Class with file change
parameters
Returns:
inserted_id
"""
file_change_event_data.machine_name = network_config[
"real_machine_identifier_name"]
file_change_event_data.file_content = binascii.b2a_base64(
open(file_change_event_data.file_path, 'rb').read()
).decode(
) if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else ""
verbose_info(messages["received_honeypot_file_change_event"].format(
file_change_event_data.file_path,
file_change_event_data.status,
file_change_event_data.module_name,
file_change_event_data.machine_name,
))
return elasticsearch_events.index(index='file_change_events',
body=file_change_event_data.__dict__)
def insert_pcap_files_to_collection(file_archive: FileArchive):
"""
Insert the pcap files containing the captured network traffic to
mongodb collection
Args:
file_archive: path of the file
Returns:
file_id
"""
verbose_info(messages["received_network_traffic_file"].format(
file_archive.file_path, file_archive.date))
file_content = binascii.b2a_base64(
open(file_archive.file_path, "rb").read()).decode()
file_md5 = hashlib.md5(file_content.encode()).hexdigest()
return elasticsearch_events.index(
index='ohp_file_archive',
body={
"md5": file_md5,
"content": file_content,
"filename": os.path.split(file_archive.file_path)[1],
"machine_name":
network_configuration()["real_machine_identifier_name"],
"date": file_archive.date,
"splitTimeout": file_archive.split_timeout
})
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent,
honeypot_events_queue: Queue):
"""
insert selected modules event to honeypot_events_queue
Args:
honeypot_event: Object of HoneypotEvent class with event parameters
honeypot_events_queue: Multiprocessing queue which stores the list of
honeypot_events in _dict_ format
Returns:
None
"""
verbose_info(messages["received_honeypot_event"].format(
honeypot_event.ip_dest, honeypot_event.port_dest,
honeypot_event.ip_src, honeypot_event.port_src,
honeypot_event.module_name, honeypot_event.machine_name))
# Get country of the source IP Address
honeypot_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_src))
# Get country of the destination IP Address
honeypot_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_dest))
honeypot_events_queue.put(honeypot_event.__dict__)
return
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
Returns:
inserted_id
"""
event_data.machine_name = \
network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(event_data.ip))
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data))
return events_data.insert_one(event_data.__dict__).inserted_id
def push_events_queues_to_database(honeypot_events_queue,
network_events_queue):
"""
Pushes all honeypot and network events collected in the
honeypot_events_queue and network_events_queue to honeypot_events
and network_events collection respectively
Args:
honeypot_events_queue: Multiprocessing queue which stores the list of
honeypot_events in _dict_ format
network_events_queue: Multiprocessing queue which stores the list of
network_events in _dict_ format
"""
if honeypot_events_queue or network_events_queue:
verbose_info(messages["submitting_events"])
# Insert all honeypot events to database (honeypot_events collection)
while not honeypot_events_queue.empty():
new_event = honeypot_events_queue.get()
elasticsearch_events.index(index='honeypot_events', body=new_event)
# Insert all network events to database (network_events collection)
while not network_events_queue.empty():
new_event = network_events_queue.get()
elasticsearch_events.index(index='network_events', body=new_event)
return
def insert_honeypot_events_data_from_module_processor(ip, module_name, date,
data):
"""
insert data which is received from honeypot modules
args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
ip, module_name,
network_configuration()["real_machine_identifier_name"], data))
return honeypot_events_data.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"data":
data,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
event_data: contain ip, module_name, machine_name, date, data
Returns:
inserted_id
"""
event_data.machine_name = network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(
event_data.ip
)
)
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data
)
)
return data_events.insert_one(event_data.__dict__).inserted_id
def insert_to_file_change_events_collection(file_change_event_data: FileEventsData):
"""
insert file change events which are obtained from ftp/ssh weak_password
module
Args:
file_change_event_data: Object of FileEventsData Class with file change
parameters
Returns:
inserted_id
"""
file_change_event_data.machine_name = network_config["real_machine_identifier_name"]
file_change_event_data.file_content = open(
file_change_event_data.file_path,
'rb'
).read() if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else ""
if is_verbose_mode():
verbose_info(
"Received honeypot file change event, file_path:{0}, status:{1}, "
"module_name:{2}, module_name:{3}, machine_name:{3}".format(
file_change_event_data.file_path,
file_change_event_data.status,
file_change_event_data.module_name,
file_change_event_data.machine_name,
)
)
return file_change_events.insert_one(file_change_event_data.__dict__).inserted_id
def insert_to_credential_events_collection(credential_event: CredentialEvent):
"""
insert credentials from honeypot events which are obtained
from the module processor to credential_event collection
Args:
credential_event: Object of CredentialEvent Class with honeypot
event credentials
Returns:
inserted_id
"""
credential_event.country = byte_to_str(
IP2Location.get_country_short(
credential_event.ip
)
)
credential_event.machine_name = network_config["real_machine_identifier_name"]
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
credential_event.ip,
credential_event.username,
credential_event.password,
credential_event.module_name,
credential_event.machine_name
)
)
return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src,
module_name, machine_name):
"""
insert selected modules event to honeypot_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
module_name: module name ran on the port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(ip_dest, port_dest, ip_src, port_src, module_name,
machine_name))
global honeypot_events_queue
honeypot_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"module_name":
module_name,
"date":
now(),
"machine_name":
machine_name,
"event_type":
"honeypot_event",
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def insert_events_in_bulk():
"""
inserts all honeypot and network events in bulk to honeypot_events and network_events collection respectively
"""
global honeypot_events_queue
global network_events_queue
if is_verbose_mode() and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
if honeypot_events_queue:
new_events = honeypot_events_queue[:]
honeypot_events_queue = []
honeypot_events.insert_many(new_events)
if network_events_queue:
new_events = network_events_queue[:]
network_events_queue = []
network_events.insert_many(new_events)
return
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
event_data: contain ip, module_name, machine_name, date, data
Returns:
inserted_id
"""
event_data.machine_name = network_config["real_machine_identifier_name"]
event_data.country_ip_src = byte_to_str(
IP2Location.get_country_short(event_data.ip_src))
verbose_info(messages["received_honeypot_data_event"].format(
event_data.ip_src, event_data.module_name, event_data.machine_name,
event_data.data))
return elasticsearch_events.index(index='data_events',
body=event_data.__dict__)
def submit_report_to_db(event):
"""
this function created to submit the generated reports into db, the files are not stored in db, just the path!
Args:
event: event log
Returns:
return True if submitted otherwise False
"""
verbose_info(messages("inserting_report_db"))
session = create_connection()
session.add(
Report(
date=event["date"],
scan_unique_id=event["scan_unique_id"],
report_path_filename=json.dumps(
event["options"]["report_path_filename"]),
options=json.dumps(event["options"]),
))
return send_submit_query(session)
def push_events_queues_to_database():
"""
Pushes all honeypot and network events collected in the
honeypot_events_queue and network_events_queue to honeypot_events
and network_events collection respectively
"""
if is_verbose_mode() and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
# Insert all honeypot events to database (honeypot_events collection)
if honeypot_events_queue:
new_events = honeypot_events_queue[:]
honeypot_events_queue.clear()
honeypot_events.insert_many(new_events)
# Insert all network events to database (network_events collection)
if network_events_queue:
new_events = network_events_queue[:]
network_events_queue.clear()
network_events.insert_many(new_events)
return
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src,
machine_name):
"""
insert other network events (port scan, etc..) to network_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
ip_dest, port_dest, ip_src, port_src, machine_name))
global network_events_queue
network_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"date":
now(),
"machine_name":
machine_name,
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def push_events_queues_to_database(honeypot_events_queue, network_events_queue):
"""
Pushes all honeypot and network events collected in the
honeypot_events_queue and network_events_queue to honeypot_events
and network_events collection respectively
"""
if is_verbose_mode() and (honeypot_events_queue or network_events_queue) \
and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
# Insert all honeypot events to database (honeypot_events collection)
while not honeypot_events_queue.empty():
new_event = honeypot_events_queue.get()
honeypot_events.insert_one(new_event)
# Insert all network events to database (network_events collection)
while not network_events_queue.empty():
new_event = network_events_queue.get()
network_events.insert_one(new_event)
return
def insert_to_credential_events_collection(credential_event: CredentialEvent):
"""
insert credentials from honeypot events which are obtained
from the module processor to credential_event collection
Args:
credential_event: Object of CredentialEvent Class with honeypot
event credentials
Returns:
inserted_id
"""
credential_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(credential_event.ip_src))
credential_event.machine_name = network_config[
"real_machine_identifier_name"]
verbose_info(messages["received_honeypot_credential_event"].format(
credential_event.ip_src, credential_event.username,
credential_event.password, credential_event.module_name,
credential_event.machine_name))
return elasticsearch_events.index(index='credential_events',
body=credential_event.__dict__)
def process_conditions(
event,
module_name,
target,
scan_unique_id,
options,
response,
process_number,
module_thread_number,
total_module_thread_number,
request_number_counter,
total_number_of_requests
):
from core.alert import (success_event_info,
verbose_info,
messages)
if 'save_to_temp_events_only' in event.get('response', ''):
from database.db import submit_temp_logs_to_db
submit_temp_logs_to_db(
{
"date": now(model=None),
"target": target,
"module_name": module_name,
"scan_unique_id": scan_unique_id,
"event_name": event['response']['save_to_temp_events_only'],
# "options": options,
"options": {},
"event": event,
"data": response
}
)
if event['response']['conditions_results'] and 'save_to_temp_events_only' not in event.get('response', ''):
from database.db import submit_logs_to_db
# remove sensitive information before submitting to db
from config import nettacker_api_config
options = copy.deepcopy(options)
for key in nettacker_api_config():
try:
del options[key]
except Exception:
continue
del event['response']['conditions']
event_request_keys = copy.deepcopy(event)
del event_request_keys['response']
submit_logs_to_db(
{
"date": now(model=None),
"target": target,
"module_name": module_name,
"scan_unique_id": scan_unique_id,
# "options": options,
"options": {},
"event": event
}
)
success_event_info(
messages("send_success_event_from_module").format(
process_number,
module_name,
target,
module_thread_number,
total_module_thread_number,
request_number_counter,
total_number_of_requests,
", ".join(
[
"{}: {}".format(
key,
event_request_keys[key]
) for key in event_request_keys
]
),
", ".join(event['response']['conditions_results'].keys())
)
)
verbose_info(
json.dumps(event)
)
return True
else:
del event['response']['conditions']
verbose_info(
messages("send_unsuccess_event_from_module").format(
process_number,
module_name,
target,
module_thread_number,
total_module_thread_number,
request_number_counter,
total_number_of_requests
)
)
verbose_info(
json.dumps(event)
)
return 'save_to_temp_events_only' in event['response']
