def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
Returns:
inserted_id
"""
event_data.machine_name = \
network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(event_data.ip))
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data))
return events_data.insert_one(event_data.__dict__).inserted_id
def insert_honeypot_events_data_from_module_processor(ip, module_name, date,
data):
"""
insert data which is received from honeypot modules
args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
ip, module_name,
network_configuration()["real_machine_identifier_name"], data))
return honeypot_events_data.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"data":
data,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
event_data: contain ip, module_name, machine_name, date, data
Returns:
inserted_id
"""
event_data.machine_name = network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(
event_data.ip
)
)
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data
)
)
return data_events.insert_one(event_data.__dict__).inserted_id
def insert_pcap_files_to_collection(file_archive: FileArchive):
"""
Insert the pcap files containing the captured network traffic to
mongodb collection
Args:
file_archive: path of the file
Returns:
file_id
"""
if is_verbose_mode():
verbose_info(
"Received network traffic file:{0}, date:{1}. "
"Inserting it in the File Archive".format(
file_archive.file_path,
file_archive.date
)
)
return ohp_file_archive_gridfs.put(
open(file_archive.file_path, "rb"),
filename=os.path.split(file_archive.file_path)[1],
machine_name=network_configuration()["real_machine_identifier_name"],
date=file_archive.date,
splitTimeout=file_archive.split_timeout
)
def insert_to_credential_events_collection(credential_event: CredentialEvent):
"""
insert credentials from honeypot events which are obtained
from the module processor to credential_event collection
Args:
credential_event: Object of CredentialEvent Class with honeypot
event credentials
Returns:
inserted_id
"""
credential_event.country = byte_to_str(
IP2Location.get_country_short(
credential_event.ip
)
)
credential_event.machine_name = network_config["real_machine_identifier_name"]
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
credential_event.ip,
credential_event.username,
credential_event.password,
credential_event.module_name,
credential_event.machine_name
)
)
return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_to_file_change_events_collection(file_change_event_data: FileEventsData):
"""
insert file change events which are obtained from ftp/ssh weak_password
module
Args:
file_change_event_data: Object of FileEventsData Class with file change
parameters
Returns:
inserted_id
"""
file_change_event_data.machine_name = network_config["real_machine_identifier_name"]
file_change_event_data.file_content = open(
file_change_event_data.file_path,
'rb'
).read() if not file_change_event_data.is_directory and file_change_event_data.status != "deleted" else ""
if is_verbose_mode():
verbose_info(
"Received honeypot file change event, file_path:{0}, status:{1}, "
"module_name:{2}, module_name:{3}, machine_name:{3}".format(
file_change_event_data.file_path,
file_change_event_data.status,
file_change_event_data.module_name,
file_change_event_data.machine_name,
)
)
return file_change_events.insert_one(file_change_event_data.__dict__).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent):
"""
insert other network events (port scan, etc..) to network_events
collection
Args:
network_event: Object of NetworkEvent Class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
network_event.ip_dest, network_event.port_dest,
network_event.ip_src, network_event.port_src,
network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.append(network_event.__dict__)
return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent):
"""
insert selected modules event to honeypot_events collection
Args:
honeypot_event: Object of HoneypotEvent class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(honeypot_event.ip_dest, honeypot_event.port_dest,
honeypot_event.ip_src, honeypot_event.port_src,
honeypot_event.module_name, honeypot_event.machine_name))
# Get country of the source IP Address
honeypot_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_src))
# Get country of the destination IP Address
honeypot_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_dest))
honeypot_events_queue.append(honeypot_event.__dict__)
return
def insert_honeypot_events_credential_from_module_processor(
ip, username, password, module_name, date):
"""
insert honeypot events which are obtained from the modules
args:
ip : client ip used for connecting to the module
username : username tried for connecting to modules
password : password tried for connecting to modules
module_name : on which module client accessed
date : datetime of the event
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
ip, username, password, module_name,
network_configuration()["real_machine_identifier_name"]))
return credential_events.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"username":
username,
"password":
password,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def create_new_images(configuration):
"""
start new images based on configuration and dockerfile
Args:
configuration: user final configuration
Returns:
True
"""
for selected_module in configuration:
# go to tmp folder to create Dockerfile and files dir
tmp_dir_name = make_tmp_thread_dir()
os.chdir(tmp_dir_name)
# create files dir
mkdir("files")
# create Dockerfile
dockerfile = open("Dockerfile", "w")
dockerfile.write(configuration[selected_module]["dockerfile"])
dockerfile.close()
# copy files
copy_dir_tree(configuration[selected_module]["files"], "files")
# create docker image
image_name = virtual_machine_name_to_container_name(
configuration[selected_module]["virtual_machine_name"],
selected_module
)
info("creating image {0}".format(image_name))
# in case if verbose mode is enabled, we will be use os.system
# instead of os.popen to show the outputs in case
# of anyone want to be aware what's happening or what's the error,
# it's a good feature for developers as well
# to create new modules
if is_verbose_mode():
os.system("docker build . -t {0}".format(image_name))
else:
os.popen("docker build . -t {0}".format(image_name)).read()
# created
info("image {0} created".format(image_name))
# go back to home directory
os.chdir("../..")
# submit tmp dir name
tmp_directories.append(tmp_dir_name)
return True
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src,
module_name, machine_name):
"""
insert selected modules event to honeypot_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
module_name: module name ran on the port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(ip_dest, port_dest, ip_src, port_src, module_name,
machine_name))
global honeypot_events_queue
honeypot_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"module_name":
module_name,
"date":
now(),
"machine_name":
machine_name,
"event_type":
"honeypot_event",
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def insert_events_in_bulk():
"""
inserts all honeypot and network events in bulk to honeypot_events and network_events collection respectively
"""
global honeypot_events_queue
global network_events_queue
if is_verbose_mode() and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
if honeypot_events_queue:
new_events = honeypot_events_queue[:]
honeypot_events_queue = []
honeypot_events.insert_many(new_events)
if network_events_queue:
new_events = network_events_queue[:]
network_events_queue = []
network_events.insert_many(new_events)
return
def verbose_info(content):
"""
build the info message, log the message in database
if requested, rewrite the thread temporary file
Args:
content: content of the message
Returns:
None
"""
if is_verbose_mode():
logger.info(content)
sys.stdout.buffer.write(
bytes(
color.color_cmd("cyan") + "[v] [{0}] ".format(now()) +
color.color_cmd("grey") + content + color.color_cmd("reset") +
"\n", "utf8"))
sys.stdout.flush()
return
def push_events_queues_to_database():
"""
Pushes all honeypot and network events collected in the
honeypot_events_queue and network_events_queue to honeypot_events
and network_events collection respectively
"""
if is_verbose_mode() and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
# Insert all honeypot events to database (honeypot_events collection)
if honeypot_events_queue:
new_events = honeypot_events_queue[:]
honeypot_events_queue.clear()
honeypot_events.insert_many(new_events)
# Insert all network events to database (network_events collection)
if network_events_queue:
new_events = network_events_queue[:]
network_events_queue.clear()
network_events.insert_many(new_events)
return
def push_events_queues_to_database(honeypot_events_queue, network_events_queue):
"""
Pushes all honeypot and network events collected in the
honeypot_events_queue and network_events_queue to honeypot_events
and network_events collection respectively
"""
if is_verbose_mode() and (honeypot_events_queue or network_events_queue) \
and (honeypot_events_queue or network_events_queue):
verbose_info("Submitting new events to database")
# Insert all honeypot events to database (honeypot_events collection)
while not honeypot_events_queue.empty():
new_event = honeypot_events_queue.get()
honeypot_events.insert_one(new_event)
# Insert all network events to database (network_events collection)
while not network_events_queue.empty():
new_event = network_events_queue.get()
network_events.insert_one(new_event)
return
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src,
machine_name):
"""
insert other network events (port scan, etc..) to network_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
ip_dest, port_dest, ip_src, port_src, machine_name))
global network_events_queue
network_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"date":
now(),
"machine_name":
machine_name,
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def test_run_with_verbose_option(self):
testargs = ['python3', 'ohp.py', '--verbose']
with patch.object(sys, 'argv', testargs):
returned_value = is_verbose_mode()
self.assertEqual(returned_value, True)
def network_traffic_capture(configuration, honeypot_events_queue,
network_events_queue, network_config):
"""
Capture network traffic and submit new network and honeypot events to the database
Args:
configuration: user final configuration
honeypot_events_queue: multiprocessing queue for storing honeypot events
network_events_queue: multiprocessing queue for storing network events
network_config: network configuration
Returns:
True
"""
info(messages["network_traffic_capture_start"])
for selected_module in configuration:
port_number = configuration[selected_module][
"real_machine_port_number"]
honeypot_ports[port_number] = selected_module
# get ip addresses
virtual_machine_ip_addresses = [
configuration[selected_module]["ip_address"]
for selected_module in configuration
]
# Ignore VM IPs + IPs in config.py
# VM = virtual machine, RM = real machine
ignore_rm_ip_addresses = network_config["ignore_real_machine_ip_address"]
ignore_vm_ip_addresses = network_config[
"ignore_virtual_machine_ip_addresses"]
# Ignore real machine IPs
ignore_ip_addresses = network_config["ignore_real_machine_ip_addresses"] \
if ignore_rm_ip_addresses else [] + virtual_machine_ip_addresses \
if ignore_vm_ip_addresses else []
ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration))
# Ignore ports
ignore_ports = network_config["ignore_real_machine_ports"]
# Display filter to be applied to the Live Captured network traffic
display_filter = ' and '.join(
['ip.src!={0} and ip.dst!={0}'.format(_) for _ in ignore_ip_addresses])
display_filter += ' and ' if ignore_ip_addresses and ignore_ports else ""
display_filter += ' and '.join([
'tcp.srcport!={0} and tcp.dstport!={0}'.format(_) for _ in ignore_ports
])
store_pcap = network_config["store_network_captured_files"]
timeout = network_config["split_pcap_file_timeout"]
# Make the pcapfiles directory for storing the Network captured files
base_dir_path = os.path.join(sys.path[0], "pcapfiles")
def packet_callback(packet):
"""
Callback function, called by apply_on_packets
Args:
packet
"""
process_packet(packet, honeypot_events_queue, network_events_queue)
# Run infinite loop and split the capture in multiple files using the timeout set
# in the network configuration
while True:
# Timestamp to be used in file name
file_timestamp = int(time.time())
generation_time = datetime.fromtimestamp(file_timestamp).strftime(
"%Y-%m-%d %H:%M:%S")
# File path of the network capture file with the timestamp
output_file_path = os.path.join(
base_dir_path, "captured-traffic-" + str(file_timestamp) + ".pcap")
if store_pcap:
info(messages["network_capture_getting_stored"].format(
output_file_path))
try:
capture = pyshark.LiveCapture(
interface='any',
display_filter=display_filter,
output_file=output_file_path if store_pcap else None)
# Debug option for pyshark capture
if is_verbose_mode():
capture.set_debug()
# Applied on every packet captured by pyshark LiveCapture
capture.apply_on_packets(packet_callback, timeout=timeout)
except get_timeout_error() as e:
force_kill_tshark()
# Catches the timeout error thrown by apply_on_packets
insert_pcap_files_to_collection(
FileArchive(output_file_path, generation_time,
timeout)) if store_pcap else e
except KeyboardInterrupt as e:
force_kill_tshark()
insert_pcap_files_to_collection(
FileArchive(output_file_path, generation_time,
timeout)) if store_pcap else e
break
except Exception as e:
force_kill_tshark()
insert_pcap_files_to_collection(
FileArchive(output_file_path, generation_time,
timeout)) if store_pcap else e
error(str(e))
break
return True
