def insert_honeypot_events_data_from_module_processor(ip, module_name, date,
data):
"""
insert data which is received from honeypot modules
args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
ip, module_name,
network_configuration()["real_machine_identifier_name"], data))
return honeypot_events_data.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"data":
data,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent,
honeypot_events_queue: Queue):
"""
insert selected modules event to honeypot_events_queue
Args:
honeypot_event: Object of HoneypotEvent class with event parameters
honeypot_events_queue: Multiprocessing queue which stores the list of
honeypot_events in _dict_ format
Returns:
None
"""
verbose_info(messages["received_honeypot_event"].format(
honeypot_event.ip_dest, honeypot_event.port_dest,
honeypot_event.ip_src, honeypot_event.port_src,
honeypot_event.module_name, honeypot_event.machine_name))
# Get country of the source IP Address
honeypot_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_src))
# Get country of the destination IP Address
honeypot_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_dest))
honeypot_events_queue.put(honeypot_event.__dict__)
return
def insert_honeypot_events_credential_from_module_processor(
ip, username, password, module_name, date):
"""
insert honeypot events which are obtained from the modules
args:
ip : client ip used for connecting to the module
username : username tried for connecting to modules
password : password tried for connecting to modules
module_name : on which module client accessed
date : datetime of the event
:return: inserted_id
"""
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
ip, username, password, module_name,
network_configuration()["real_machine_identifier_name"]))
return credential_events.insert_one({
"ip_dest":
byte_to_str(ip),
"module_name":
module_name,
"date":
date,
"username":
username,
"password":
password,
"country":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip))),
"machine_name":
network_configuration()["real_machine_identifier_name"]
}).inserted_id
def insert_to_network_events_queue(network_event: NetworkEvent,
network_events_queue: Queue):
"""
insert other network events (port scan, etc..) to network_events_queue
Args:
network_event: Object of NetworkEvent Class with event parameters
network_events_queue: Multiprocessing queue which stores the list of
network_events in _dict_ format
Returns:
None
"""
verbose_info(messages["received_network_event"].format(
network_event.ip_dest, network_event.port_dest, network_event.ip_src,
network_event.port_src, network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.put(network_event.__dict__)
return
def insert_to_network_events_queue(network_event: NetworkEvent):
"""
insert other network events (port scan, etc..) to network_events
collection
Args:
network_event: Object of NetworkEvent Class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
network_event.ip_dest, network_event.port_dest,
network_event.ip_src, network_event.port_src,
network_event.machine_name))
# Get country of the source IP Address
network_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(network_event.ip_src))
# Get country of the destination IP Address
network_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(network_event.ip_dest))
network_events_queue.append(network_event.__dict__)
return
def insert_to_honeypot_events_queue(honeypot_event: HoneypotEvent):
"""
insert selected modules event to honeypot_events collection
Args:
honeypot_event: Object of HoneypotEvent class with event parameters
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(honeypot_event.ip_dest, honeypot_event.port_dest,
honeypot_event.ip_src, honeypot_event.port_src,
honeypot_event.module_name, honeypot_event.machine_name))
# Get country of the source IP Address
honeypot_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_src))
# Get country of the destination IP Address
honeypot_event.country_ip_dest = byte_to_str(
IP2Location.get_country_short(honeypot_event.ip_dest))
honeypot_events_queue.append(honeypot_event.__dict__)
return
def on_any_event(self, event):
if not (event.event_type == 'modified' and event.is_directory) \
and not is_excluded(event.src_path, self.EXCLUDES):
insert_to_file_change_events_collection(
FileEventsData(file_path=byte_to_str(event.src_path),
status=byte_to_str(event.event_type),
module_name=self.module_name,
date=now(),
is_directory=event.is_directory))
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
event_data: contain ip, module_name, machine_name, date, data
Returns:
inserted_id
"""
event_data.machine_name = network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(
event_data.ip
)
)
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(
event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data
)
)
return data_events.insert_one(event_data.__dict__).inserted_id
def insert_to_credential_events_collection(credential_event: CredentialEvent):
"""
insert credentials from honeypot events which are obtained
from the module processor to credential_event collection
Args:
credential_event: Object of CredentialEvent Class with honeypot
event credentials
Returns:
inserted_id
"""
credential_event.country = byte_to_str(
IP2Location.get_country_short(
credential_event.ip
)
)
credential_event.machine_name = network_config["real_machine_identifier_name"]
if is_verbose_mode():
verbose_info(
"Received honeypot credential event, ip_dest:{0}, username:{1}, "
"password:{2}, module_name:{3}, machine_name:{4}".format(
credential_event.ip,
credential_event.username,
credential_event.password,
credential_event.module_name,
credential_event.machine_name
)
)
return credential_events.insert_one(credential_event.__dict__).inserted_id
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
ip : client ip used for putting the data
module_name : on which module client accessed
date : datetime of the events
data : Data which is obtained from the client
Returns:
inserted_id
"""
event_data.machine_name = \
network_config["real_machine_identifier_name"]
event_data.country = byte_to_str(
IP2Location.get_country_short(event_data.ip))
if is_verbose_mode():
verbose_info(
"Received honeypot data event, ip_dest:{0}, module_name:{1}, "
"machine_name:{2}, data:{3}".format(event_data.ip,
event_data.module_name,
event_data.machine_name,
event_data.data))
return events_data.insert_one(event_data.__dict__).inserted_id
def insert_to_events_data_collection(event_data: EventData):
"""
Insert data collected from module processors of modules such as-
ICS module
Args:
event_data: contain ip, module_name, machine_name, date, data
Returns:
inserted_id
"""
event_data.machine_name = network_config["real_machine_identifier_name"]
event_data.country_ip_src = byte_to_str(
IP2Location.get_country_short(event_data.ip_src))
verbose_info(messages["received_honeypot_data_event"].format(
event_data.ip_src, event_data.module_name, event_data.machine_name,
event_data.data))
return elasticsearch_events.index(index='data_events',
body=event_data.__dict__)
def insert_selected_modules_network_event(ip_dest, port_dest, ip_src, port_src,
module_name, machine_name):
"""
insert selected modules event to honeypot_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
module_name: module name ran on the port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info(
"Received honeypot event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, module_name:{4}, machine_name:{5}".
format(ip_dest, port_dest, ip_src, port_src, module_name,
machine_name))
global honeypot_events_queue
honeypot_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"module_name":
module_name,
"date":
now(),
"machine_name":
machine_name,
"event_type":
"honeypot_event",
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def insert_to_credential_events_collection(credential_event: CredentialEvent):
"""
insert credentials from honeypot events which are obtained
from the module processor to credential_event collection
Args:
credential_event: Object of CredentialEvent Class with honeypot
event credentials
Returns:
inserted_id
"""
credential_event.country_ip_src = byte_to_str(
IP2Location.get_country_short(credential_event.ip_src))
credential_event.machine_name = network_config[
"real_machine_identifier_name"]
verbose_info(messages["received_honeypot_credential_event"].format(
credential_event.ip_src, credential_event.username,
credential_event.password, credential_event.module_name,
credential_event.machine_name))
return elasticsearch_events.index(index='credential_events',
body=credential_event.__dict__)
def insert_other_network_event(ip_dest, port_dest, ip_src, port_src,
machine_name):
"""
insert other network events (port scan, etc..) to network_events collection
Args:
ip_dest: dest ip (machine)
port_dest: dest port (machine)
ip_src: src ip
port_src: src port
machine_name: real machine name
Returns:
ObjectId(inserted_id)
"""
if is_verbose_mode():
verbose_info("Received network event, ip_dest:{0}, port_dest:{1}, "
"ip_src:{2}, port_src:{3}, machine_name:{4}".format(
ip_dest, port_dest, ip_src, port_src, machine_name))
global network_events_queue
network_events_queue.append({
"ip_dest":
byte_to_str(ip_dest),
"port_dest":
int(port_dest),
"ip_src":
byte_to_str(ip_src),
"port_src":
int(port_src),
"date":
now(),
"machine_name":
machine_name,
"country_ip_src":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_src))),
"country_ip_dest":
byte_to_str(IP2Location.get_country_short(byte_to_str(ip_dest)))
})
return
def test_byte_to_str(self):
content = b"hello"
self.assertIsInstance(byte_to_str(content), str)
def new_network_events(configuration):
"""
get and submit new network events
Args:
configuration: user final configuration
Returns:
True
"""
info("new_network_events thread started")
# honeypot ports
honeypot_ports = []
for selected_module in configuration:
honeypot_ports.append(
configuration[selected_module]["real_machine_port_number"])
# set machine name
machine_name = network_configuration()["real_machine_identifier_name"]
# get ip addresses
virtual_machine_ip_addresses = [
configuration[selected_module]["ip_address"]
for selected_module in configuration
]
# ignore vm ips + ips in config.py
ignore_ip_addresses = network_configuration()["ignore_real_machine_ip_addresses"] \
if network_configuration()["ignore_real_machine_ip_address"] else [] + virtual_machine_ip_addresses \
if network_configuration()["ignore_virtual_machine_ip_addresses"] else []
ignore_ip_addresses.extend(get_gateway_ip_addresses(configuration))
# ignore ports
ignore_ports = network_configuration()["ignore_real_machine_ports"]
# start tshark to capture network
# tshark -Y "ip.dst != 192.168.1.1" -T fields -e ip.dst -e tcp.srcport
run_tshark = ["tshark", "-l", "-V"]
run_tshark.extend(ignore_ip_addresses_rule_generator(ignore_ip_addresses))
run_tshark.extend([
"-T", "fields", "-e", "ip.dst", "-e", "ip.src", "-e", "tcp.dstport",
"-e", "tcp.srcport", "-ni", "any"
])
process = subprocess.Popen(run_tshark,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# wait 3 seconds if process terminated?
time.sleep(3)
if process.poll() is not None:
exit_failure("tshark couldn't capture network, maybe run as root!")
# todo: replace tshark with python port sniffing - e.g https://www.binarytides.com/python-packet-sniffer-code-linux/
# it will be easier to apply filters and analysis packets with python
# if it requires to be run as root, please add a uid checker in framework startup
# readline timeout bug fix: https://stackoverflow.com/a/10759061
pull_object = select.poll()
pull_object.register(process.stdout, select.POLLIN)
# while True, read tshark output
try:
while True:
if pull_object.poll(0):
line = process.stdout.readline()
# check if new IP and Port printed
if len(line) > 0:
# split the IP and Port
try:
line = line.rsplit()
ip_dest = byte_to_str(line[0])
ip_src = byte_to_str(line[1])
port_dest = int(line[2])
port_src = int(line[3])
if (netaddr.valid_ipv4(ip_dest) or netaddr.valid_ipv6(ip_dest)) \
and ip_dest not in ignore_ip_addresses \
and ip_src not in ignore_ip_addresses \
and port_dest not in ignore_ports \
and port_src not in ignore_ports:
# ignored ip addresses and ports in python - fix later
# check if the port is in selected module
if port_dest in honeypot_ports or port_src in honeypot_ports:
if port_dest in honeypot_ports:
insert_selected_modules_network_event(
ip_dest, port_dest, ip_src, port_src,
selected_module, machine_name)
else:
insert_other_network_event(
ip_dest, port_dest, ip_src, port_src,
machine_name)
except Exception as _:
del _
# check if event shows an IP
time.sleep(0.001)
# todo: is sleep(0.001) fastest/best? it means it could get 1000 packets per second (maximum) from tshark
# how could we prevent the DDoS attacks in here and avoid submitting in MongoDB? should we?
except Exception as _:
del _
return True
