def insert_honeypot_events_data_from_module_processor(ip, module_name, date,
data):
"""
insert data which is received from honeypot modules
args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
ip, module_name,
network_configuration()["real_machine_identifier_name"], data))
return honeypot_events_data.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"data":
data,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_honeypot_events_credential_from_module_processor(
ip, username, password, module_name, date):
"""
insert honeypot events which are obtained from the modules
args:
ip : client ip used for connecting to the module
username : username tried for connecting to modules
password : password tried for connecting to modules
module_name : on which module client accessed
date : datetime of the event
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
ip, username, password, module_name,
network_configuration()["real_machine_identifier_name"]))
return credential_events.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"username":
username,
"password":
password,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_honeypot_events_from_module_processor(ip, username, password,
module_name, date):
"""
insert honeypot events which are obtained from the modules
args:
ip : client ip used for connecting to the module
username : username tried for connecting to modules
password : password tried for connecting to modules
module_name : on which module client accessed
date : datetime of the event
"""
return credential_events.insert_one({
"ip":
ip,
"module_name":
module_name,
"date":
date,
"username":
username,
"password":
password,
"country":
str(IP2Location.get_country_short(ip).decode()),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_pcap_files_to_collection(file_archive: FileArchive):
"""
Insert the pcap files containing the captured network traffic to
mongodb collection
Args:
file_archive: path of the file
Returns:
file_id
"""
verbose_info(messages["received_network_traffic_file"].format(
file_archive.file_path, file_archive.date))
file_content = binascii.b2a_base64(
open(file_archive.file_path, "rb").read()).decode()
file_md5 = hashlib.md5(file_content.encode()).hexdigest()
return elasticsearch_events.index(
index='ohp_file_archive',
body={
"md5": file_md5,
"content": file_content,
"filename": os.path.split(file_archive.file_path)[1],
"machine_name":
network_configuration()["real_machine_identifier_name"],
"date": file_archive.date,
"splitTimeout": file_archive.split_timeout
})
def insert_pcap_files_to_collection(file_archive: FileArchive):
"""
Insert the pcap files containing the captured network traffic to
mongodb collection
Args:
file_archive: path of the file
Returns:
file_id
"""
if is_verbose_mode():
verbose_info(
"Received network traffic file:{0}, date:{1}. "
"Inserting it in the File Archive".format(
file_archive.file_path,
file_archive.date
)
)
return ohp_file_archive_gridfs.put(
open(file_archive.file_path, "rb"),
filename=os.path.split(file_archive.file_path)[1],
machine_name=network_configuration()["real_machine_identifier_name"],
date=file_archive.date,
splitTimeout=file_archive.split_timeout
)
def process_packet(packet, honeypot_events_queue, network_events_queue):
"""
Callback function called from the apply_on_packets function.
Args:
packet: Packet live captured by pyshark
honeypot_events_queue: multiprocessing queue for storing honeypot events
network_events_queue: multiprocessing queue for storing network events
"""
# set machine name
machine_name = network_configuration()["real_machine_identifier_name"]
try:
# Check if packet contains IP layer
if "IP" in packet:
ip_dest = packet.ip.dst
ip_src = packet.ip.src
protocol = protocol_table[int(packet.ip.proto)]
port_dest = int()
port_src = int()
# Check packet protocol and if it contains a layer with the same
# name
if protocol == "TCP" and "TCP" in packet:
port_dest = int(packet.tcp.dstport)
port_src = int(packet.tcp.srcport)
elif protocol == "UDP" and "UDP" in packet:
port_dest = int(packet.udp.dstport)
port_src = int(packet.udp.srcport)
if netaddr.valid_ipv4(ip_dest) or netaddr.valid_ipv6(ip_dest):
# ignored ip addresses and ports in python - fix later
# check if the port is in selected module
insert_to_honeypot_events_queue(
HoneypotEvent(
ip_dest,
port_dest,
ip_src,
port_src,
protocol,
honeypot_ports[port_dest if port_dest in honeypot_ports.keys() else port_src],
machine_name
),
honeypot_events_queue
) if port_dest in honeypot_ports.keys() or port_src in honeypot_ports \
else insert_to_network_events_queue(
NetworkEvent(
ip_dest,
port_dest,
ip_src,
port_src,
protocol,
machine_name
),
network_events_queue
)
except Exception as _e:
del _e
def set_network_configuration(argv_options):
"""
Set network configuration based on user selections
Args:
argv_options
Returns:
network_config
"""
network_config = network_configuration()
# Set the values of the network configuration based on CLI input
network_config["store_network_captured_files"] = argv_options.store_pcap
network_config["split_pcap_file_timeout"] = argv_options.timeout_value
return network_config
def port_is_reserved(real_machine_port):
"""
check if port is reserved
Args:
real_machine_port: port number
Returns:
True or False
"""
try:
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.bind((network_configuration()["real_machine_ip_address"], real_machine_port))
tcp.close()
return False
except Exception as _:
return True
def insert_honeypot_events_data_from_module_processor(ip, module_name, date,
data):
"""
insert data which is recieved from honeypot modules
args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
"""
return honeypot_events_data.insert_one({
"ip":
ip,
"module_name":
module_name,
"date":
date,
"data":
data,
"country":
str(IP2Location.get_country_short(ip).decode()),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def new_network_events(configuration):
"""
get and submit new network events
Args:
configuration: user final configuration
Returns:
True
"""
info("new_network_events thread started")
# set machine name
machine_name = network_configuration()["real_machine_identifier_name"]
# get ip addresses
virtual_machine_ip_addresses = [
configuration[selected_module]["ip_address"]
for selected_module in configuration
]
# ignore vm ips + ips in config.py
ignore_ip_addresses = network_configuration(
)["ignore_real_machine_ip_addresses"] + virtual_machine_ip_addresses
ignore_ip_addresses.append(
network_configuration()["real_machine_ip_address"])
ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration))
# ignore ports
ignore_ports = network_configuration()["ignore_real_machine_ports"]
# start tshark to capture network
# tshark -Y "ip.dst != 192.168.1.1" -T fields -e ip.dst -e tcp.srcport
run_tshark = ["tshark"]
run_tshark.extend(ignore_ip_addresses_rule_generator(ignore_ip_addresses))
run_tshark.extend(
["-T", "fields", "-e", "ip.dst", "-e", "tcp.srcport", "-ni", "any"])
process = subprocess.Popen(run_tshark,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# todo: replace tshark with python port sniffing - e.g https://www.binarytides.com/python-packet-sniffer-code-linux/
# it will be easier to apply filters and analysis packets with python
# if it requires to be run as root, please add a uid checker in framework startup
# readline timeout bug fix: https://stackoverflow.com/a/10759061
pull_object = select.poll()
pull_object.register(process.stdout, select.POLLIN)
# while True, read tshark output
try:
while True:
if pull_object.poll(0):
line = process.stdout.readline()
# check if new IP and Port printed
if len(line) > 0:
# split the IP and Port
try:
ip, port = str(line.rsplit()[0].decode()), int(
line.rsplit()[1])
except Exception as _:
ip, port = None, None
# check if event shows an IP
if (netaddr.valid_ipv4(ip) or netaddr.valid_ipv6(ip)) \
and ip not in ignore_ip_addresses \
and port not in ignore_ports: # ignored ip addresses and ports in python - fix later
# check if the port is in selected module
inserted_flag = True
for selected_module in configuration:
if port == configuration[selected_module][
"real_machine_port_number"]:
# insert honeypot event (selected module)
insert_selected_modules_network_event(
ip, port, selected_module, machine_name)
inserted_flag = False
break
if inserted_flag:
# insert common network event
insert_other_network_event(ip, port, machine_name)
time.sleep(0.001)
# todo: is sleep(0.001) fastest/best? it means it could get 1000 packets per second (maximum) from tshark
# how could we prevent the DDoS attacks in here and avoid submitting in MongoDB? should we?
except Exception as _:
del _
return True
def new_network_events(configuration):
"""
get and submit new network events
Args:
configuration: user final configuration
Returns:
True
"""
info("new_network_events thread started")
# honeypot ports
honeypot_ports = []
virtual_machine_ip_addresses = []
network_config = network_configuration()
for selected_module in configuration:
port_number = configuration[selected_module]["real_machine_port_number"]
ip_address = configuration[selected_module]["ip_address"]
honeypot_ports.append(port_number)
# get ip addresses
virtual_machine_ip_addresses.append(ip_address)
# set machine name
machine_name = network_config["real_machine_identifier_name"]
# ignore vm ips + ips in config.py
# vm = virtual machine, rm = real machine
ignore_rm_ip_addresses = network_config["ignore_real_machine_ip_address"]
ignore_vm_ip_addresses = network_config["ignore_virtual_machine_ip_addresses"]
ignore_ip_addresses = network_config["ignore_real_machine_ip_addresses"] \
if ignore_rm_ip_addresses else [] + virtual_machine_ip_addresses \
if ignore_vm_ip_addresses else []
ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration))
# ign
# ore ports
ignore_ports = network_config["ignore_real_machine_ports"]
# start tshark to capture network
# tshark -Y "ip.dst != 192.168.1.1" -T fields -e ip.dst -e tcp.srcport
run_tshark = ["tshark", "-l", "-V"]
run_tshark.extend(ignore_ip_addresses_rule_generator(ignore_ip_addresses))
run_tshark.extend(
[
"-T", "fields", "-e", "ip.dst", "-e", "ip.src",
"-e", "tcp.dstport", "-e", "tcp.srcport", "-ni", "any"
]
)
process = subprocess.Popen(
run_tshark,
stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
# wait 3 seconds if process terminated?
time.sleep(3)
if process.poll() is not None:
exit_failure("tshark couldn't capture network, maybe run as root!")
# todo: replace tshark with python port sniffing -
# e.g https://www.binarytides.com/python-packet-sniffer-code-linux/
# it will be easier to apply filters and analysis packets with python
# if it requires to be run as root,
# please add a uid checker in framework startup
# readline timeout bug fix: https://stackoverflow.com/a/10759061
pull_object = select.poll()
pull_object.register(process.stdout, select.POLLIN)
# while True, read tshark output
try:
while True:
if pull_object.poll(0):
line = process.stdout.readline()
# check if new IP and Port printed
if len(line) > 0:
# split the IP and Port
try:
line = line.rsplit()
ip_dest = byte_to_str(line[0])
ip_src = byte_to_str(line[1])
port_dest = int(line[2])
port_src = int(line[3])
if (netaddr.valid_ipv4(ip_dest) or
netaddr.valid_ipv6(ip_dest)) \
and ip_dest not in ignore_ip_addresses \
and ip_src not in ignore_ip_addresses \
and port_dest not in ignore_ports \
and port_src not in ignore_ports:
# ignored ip addresses and ports in python -fix later
# check if the port is in selected module
if (port_dest in honeypot_ports or
port_src in honeypot_ports):
if port_dest in honeypot_ports:
insert_to_honeypot_events_queue(
HoneypotEvent(
ip_dest=ip_dest,
port_dest=port_dest,
ip_src=ip_src,
port_src=port_src,
module_name=selected_module,
machine_name=machine_name
)
)
else:
insert_to_network_events_queue(
NetworkEvent(
ip_dest=ip_dest,
port_dest=port_dest,
ip_src=ip_src,
port_src=port_src,
machine_name=machine_name
)
)
except Exception:
pass
# check if event shows an IP
time.sleep(0.001)
# todo: is sleep(0.001) fastest/best?
# it means it could get 1000 packets per second(maximum) from
# tshark
# how could we prevent the DDoS attacks in here
# and avoid submitting in MongoDB? should we?
except Exception as _:
del _
return True
import binascii
from multiprocessing import Queue
from config import api_configuration, network_configuration
from core.alert import verbose_info
from core.compatible import byte_to_str
from database.datatypes import (CredentialEvent, HoneypotEvent, EventData,
NetworkEvent, FileEventsData, FileArchive,
elastic_search_types)
from lib.ip2location import IP2Location
from api.database_queries import event_types
from core.messages import load_messages
api_config = api_configuration()
network_config = network_configuration()
messages = load_messages().message_contents
# Event index connections
elasticsearch_events = elasticsearch.Elasticsearch(
api_config["api_database"],
http_auth=api_config["api_database_http_auth"],
)
event_types_elastic = event_types.copy()
del event_types_elastic['all']
# Event queues
honeypot_events_queue = list()
network_events_queue = list()
IP2Location = IP2Location.IP2Location(
