def insert_to_network_events_queue(network_event: NetworkEvent,
network_events_queue: Queue):
"""
insert other network events (port scan, etc..) to network_events_queue
Args:
network_event: Object of NetworkEvent Class with event parameters
network_events_queue: Multiprocessing queue which stores the list of
network_events in _dict_ format
Returns:
None
"""
verbose_info(messages["received_network_event"].format(
network_event.ip_dest, network_event.port_dest, network_event.ip_src,
network_event.port_src, network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.put(network_event.__dict__)
return
def insert_to_network_events_queue(network_event: NetworkEvent):
"""
insert other network events (port scan, etc..) to network_events
collection
Args:
network_event: Object of NetworkEvent Class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
network_event.ip_dest, network_event.port_dest,
network_event.ip_src, network_event.port_src,
network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.append(network_event.__dict__)
return
def process_packet(packet, honeypot_events_queue, network_events_queue):
"""
Callback function called from the apply_on_packets function.
Args:
packet: Packet live captured by pyshark
honeypot_events_queue: multiprocessing queue for storing honeypot events
network_events_queue: multiprocessing queue for storing network events
"""
# set machine name
machine_name = network_configuration()["real_machine_identifier_name"]
try:
# Check if packet contains IP layer
if "IP" in packet:
ip_dest = packet.ip.dst
ip_src = packet.ip.src
protocol = protocol_table[int(packet.ip.proto)]
port_dest = int()
port_src = int()
# Check packet protocol and if it contains a layer with the same
# name
if protocol == "TCP" and "TCP" in packet:
port_dest = int(packet.tcp.dstport)
port_src = int(packet.tcp.srcport)
elif protocol == "UDP" and "UDP" in packet:
port_dest = int(packet.udp.dstport)
port_src = int(packet.udp.srcport)
if netaddr.valid_ipv4(ip_dest) or netaddr.valid_ipv6(ip_dest):
# ignored ip addresses and ports in python - fix later
# check if the port is in selected module
insert_to_honeypot_events_queue(
HoneypotEvent(
ip_dest,
port_dest,
ip_src,
port_src,
protocol,
honeypot_ports[port_dest if port_dest in honeypot_ports.keys() else port_src],
machine_name
),
honeypot_events_queue
) if port_dest in honeypot_ports.keys() or port_src in honeypot_ports \
else insert_to_network_events_queue(
NetworkEvent(
ip_dest,
port_dest,
ip_src,
port_src,
protocol,
machine_name
),
network_events_queue
)
except Exception as _e:
del _e
def test_push_event_queues_to_db(self):
"""
Test pushing Honeypot and network events from queues
to database.
"""
honeypot_event = HoneypotEvent(
ip_dest="11.22.33.44",
port_dest=80,
ip_src="12.23.34.45",
port_src=1010,
protocol='TCP',
module_name="http/basic_auth_weak_password",
machine_name="stockholm_server_1"
)
network_event = NetworkEvent(
ip_dest="13.14.15.16",
port_dest=8090,
ip_src="22.33.44.55",
port_src=1100,
protocol='UDP',
machine_name="stockholm_server_1"
)
honeypot_events_queue = Queue()
network_events_queue = Queue()
# Insert events to queues
insert_to_honeypot_events_queue(honeypot_event, honeypot_events_queue)
insert_to_network_events_queue(network_event, network_events_queue)
push_events_queues_to_database(honeypot_events_queue, network_events_queue)
# Find the records in the DB
honeypot_record = honeypot_events.find_one(honeypot_event.__dict__)
network_record = network_events.find_one(network_event.__dict__)
# wait for queue to be empty
time.sleep(5)
# Compare the record found in the DB with the one pushed
self.assertEqual(honeypot_record["ip_src"], honeypot_event.ip_src)
self.assertEqual(honeypot_record["ip_dest"], honeypot_event.ip_dest)
self.assertEqual(network_record["ip_src"], network_event.ip_src)
self.assertEqual(network_record["ip_dest"], network_event.ip_dest)
# Delete test events from the database
honeypot_events.delete_one(honeypot_event.__dict__)
network_events.delete_one(network_event.__dict__)
def test_push_event_queues_to_db(self):
"""
Test pushing Honeypot and network events from queues
to database.
"""
honeypot_event = HoneypotEvent(
ip_dest="11.22.33.44",
port_dest=80,
ip_src="12.23.34.45",
port_src=1010,
protocol='TCP',
module_name="http/basic_auth_weak_password",
machine_name="stockholm_server_1"
)
network_event = NetworkEvent(
ip_dest="13.14.15.16",
port_dest=8090,
ip_src="22.33.44.55",
port_src=1100,
protocol='UDP',
machine_name="stockholm_server_1"
)
honeypot_events_queue = Queue()
network_events_queue = Queue()
# Insert events to queues
insert_to_honeypot_events_queue(honeypot_event, honeypot_events_queue)
insert_to_network_events_queue(network_event, network_events_queue)
push_events_queues_to_database(honeypot_events_queue, network_events_queue)
time.sleep(1)
honeypot_records = connector.elasticsearch_events.search(
index='honeypot_events',
body=filter_by_fields('11.22.33.44', ['ip_dest'])
)['hits']['hits']
network_records = connector.elasticsearch_events.search(
index='network_events',
body=filter_by_fields('13.14.15.16', ['ip_dest'])
)['hits']['hits']
# Find the records in the DB
honeypot_record = honeypot_records[0]['_source']
network_record = network_records[0]['_source']
self.assertGreater(len(honeypot_records), 0)
self.assertGreater(len(network_records), 0)
# Compare the record found in the DB with the one pushed
self.assertEqual(honeypot_record["ip_src"], honeypot_event.ip_src)
self.assertEqual(honeypot_record["ip_dest"], honeypot_event.ip_dest)
self.assertEqual(network_record["ip_src"], network_event.ip_src)
self.assertEqual(network_record["ip_dest"], network_event.ip_dest)
# Delete test events from the database
connector.elasticsearch_events.delete(
index='honeypot_events',
id=honeypot_records[0]["_id"]
)
connector.elasticsearch_events.delete(
index='network_events',
id=network_records[0]["_id"]
)
def new_network_events(configuration):
"""
get and submit new network events
Args:
configuration: user final configuration
Returns:
True
"""
info("new_network_events thread started")
# honeypot ports
honeypot_ports = []
virtual_machine_ip_addresses = []
network_config = network_configuration()
for selected_module in configuration:
port_number = configuration[selected_module]["real_machine_port_number"]
ip_address = configuration[selected_module]["ip_address"]
honeypot_ports.append(port_number)
# get ip addresses
virtual_machine_ip_addresses.append(ip_address)
# set machine name
machine_name = network_config["real_machine_identifier_name"]
# ignore vm ips + ips in config.py
# vm = virtual machine, rm = real machine
ignore_rm_ip_addresses = network_config["ignore_real_machine_ip_address"]
ignore_vm_ip_addresses = network_config["ignore_virtual_machine_ip_addresses"]
ignore_ip_addresses = network_config["ignore_real_machine_ip_addresses"] \
if ignore_rm_ip_addresses else [] + virtual_machine_ip_addresses \
if ignore_vm_ip_addresses else []
ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration))
# ign
# ore ports
ignore_ports = network_config["ignore_real_machine_ports"]
# start tshark to capture network
# tshark -Y "ip.dst != 192.168.1.1" -T fields -e ip.dst -e tcp.srcport
run_tshark = ["tshark", "-l", "-V"]
run_tshark.extend(ignore_ip_addresses_rule_generator(ignore_ip_addresses))
run_tshark.extend(
[
"-T", "fields", "-e", "ip.dst", "-e", "ip.src",
"-e", "tcp.dstport", "-e", "tcp.srcport", "-ni", "any"
]
)
process = subprocess.Popen(
run_tshark,
stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
# wait 3 seconds if process terminated?
time.sleep(3)
if process.poll() is not None:
exit_failure("tshark couldn't capture network, maybe run as root!")
# todo: replace tshark with python port sniffing -
# e.g https://www.binarytides.com/python-packet-sniffer-code-linux/
# it will be easier to apply filters and analysis packets with python
# if it requires to be run as root,
# please add a uid checker in framework startup
# readline timeout bug fix: https://stackoverflow.com/a/10759061
pull_object = select.poll()
pull_object.register(process.stdout, select.POLLIN)
# while True, read tshark output
try:
while True:
if pull_object.poll(0):
line = process.stdout.readline()
# check if new IP and Port printed
if len(line) > 0:
# split the IP and Port
try:
line = line.rsplit()
ip_dest = byte_to_str(line[0])
ip_src = byte_to_str(line[1])
port_dest = int(line[2])
port_src = int(line[3])
if (netaddr.valid_ipv4(ip_dest) or
netaddr.valid_ipv6(ip_dest)) \
and ip_dest not in ignore_ip_addresses \
and ip_src not in ignore_ip_addresses \
and port_dest not in ignore_ports \
and port_src not in ignore_ports:
# ignored ip addresses and ports in python -fix later
# check if the port is in selected module
if (port_dest in honeypot_ports or
port_src in honeypot_ports):
if port_dest in honeypot_ports:
insert_to_honeypot_events_queue(
HoneypotEvent(
ip_dest=ip_dest,
port_dest=port_dest,
ip_src=ip_src,
port_src=port_src,
module_name=selected_module,
machine_name=machine_name
)
)
else:
insert_to_network_events_queue(
NetworkEvent(
ip_dest=ip_dest,
port_dest=port_dest,
ip_src=ip_src,
port_src=port_src,
machine_name=machine_name
)
)
except Exception:
pass
# check if event shows an IP
time.sleep(0.001)
# todo: is sleep(0.001) fastest/best?
# it means it could get 1000 packets per second(maximum) from
# tshark
# how could we prevent the DDoS attacks in here
# and avoid submitting in MongoDB? should we?
except Exception as _:
del _
return True