def __init__(self, *, cfirewall):
self._initialize = Initialize(Log, 'FirewallControl')
self.BEFORE = {}
self.MAIN = {}
self.AFTER = {}
# reference to extension CFirewall, which handles nfqueue and initial packet rcv.
# we will use this reference to modify firewall rules which will be internally accessed
# by the inspection function callbacks
self.cfirewall = cfirewall
class Configuration:
_service_setup = False
def __init__(self, name):
self.Initialize = Initialize(Log, name)
@classmethod
def service_setup(cls, SyslogService):
'''start threads for tasks required by the syslog service. blocking until settings are loaded/initialized.'''
if (cls._service_setup):
raise RuntimeError('service setup should only be called once.')
cls._service_setup = True
self = cls(SyslogService.__name__)
self.SyslogService = SyslogService
self.Initialize.wait_for_threads(count=1)
threading.Thread(target=self.get_settings).start()
@cfg_read_poller('syslog_client')
def get_settings(self, cfg_file):
syslog = load_configuration(cfg_file)['syslog']
SyslogService = self.SyslogService
SyslogService.syslog_enabled = syslog['enabled']
SyslogService.syslog_protocol = syslog['protocol']
SyslogService.tls_enabled = syslog['tls']['enabled']
SyslogService.self_signed_cert = syslog['tls']['self_signed']
SyslogService.tcp_fallback = syslog['tcp']['fallback']
SyslogService.udp_fallback = syslog['udp']['fallback']
syslog_servers = syslog['servers']
# if service is started without servers configured we will return here.
if not syslog_servers: return
names = ['primary', 'secondary']
with SyslogService.server_lock:
for name, cfg_server, mem_server in zip(names, syslog_servers.values(), SyslogService.syslog_servers):
if (cfg_server['ip_address'] == mem_server.get('ip')): continue
getattr(SyslogService.syslog_servers, name).update({
'ip': syslog_servers[name]['ip_address'],
PROTO.UDP: True, PROTO.TCP: True, PROTO.DNS_TLS: True
})
def get_interface_settings(self):
interface_settings = load_configuration('config.json')
self.lan_int = interface_settings['settings']['interface']['inside']
class FirewallControl:
__slots__ = (
'cfirewall',
'_initialize',
# firewall sections (hierarchy)
# NOTE: these are used primarily to detect config changes to prevent
# the amount of work/ data conversions that need to be done to load
# the settings into C data structures.
'BEFORE',
'MAIN',
'AFTER')
def __init__(self, *, cfirewall):
self._initialize = Initialize(Log, 'FirewallControl')
self.BEFORE = {}
self.MAIN = {}
self.AFTER = {}
# reference to extension CFirewall, which handles nfqueue and initial packet rcv.
# we will use this reference to modify firewall rules which will be internally accessed
# by the inspection function callbacks
self.cfirewall = cfirewall
# threads will be started here.
def run(self):
self._init_system_rules()
threading.Thread(target=self.monitor_zones).start()
threading.Thread(target=self.monitor_rules).start()
self._initialize.wait_for_threads(count=2)
@cfg_read_poller('zone_map', folder='iptables')
# zone int values are arbitrary / randomly selected on zone creation.
def monitor_zones(self, zone_map):
'''calls to Cython are made from within this method block. the GIL must be manually acquired on the Cython
side or the Python interpreter will crash. Monitors the firewall zone file for changes and loads updates to
cfirewall.'''
dnx_zones = load_configuration(zone_map,
filepath='dnx_system/iptables')
# converting list to python array, then sending to Cython to modify C array.
# this format is required due to transitioning between python and C. python arrays are
# compatible in C via memory views and Cython can handle the initial list.
dnx_zones = array('i', dnx_zones['map'])
print(f'sending zones to CFirewall: {dnx_zones}')
# NOTE: gil must be held on the other side of this call
error = self.cfirewall.update_zones(dnx_zones)
if (error):
pass # TODO: do something here
self._initialize.done()
@cfg_read_poller('firewall_active', folder='iptables')
def monitor_rules(self, fw_rules):
'''calls to Cython are made from within this method block. the GIL must be manually acquired on the Cython
side or the Python interpreter will crash. Monitors the active firewall rules file for changes and loads
updates to cfirewall.'''
dnx_fw = load_configuration(fw_rules, filepath='dnx_system/iptables')
# splitting out sections then determine which one has changed. this is to reduce
# amount of work done on the C side. not for performance, but more for ease of programming.
# NOTE: index 1 start is needed because SYSTEM rules are held at index 0.
for i, section in enumerate(['BEFORE', 'MAIN', 'AFTER'], 1):
current_section = getattr(self, section)
new_section = dnx_fw[section]
# unchanged ruleset
if (current_section == new_section): continue
# updating ruleset to reflect changes
setattr(self, section, new_section)
# converting dict to list and each rule into a py array. this format is required due to
# transitioning between python and C. python arrays are compatible in C via memory views
# and Cython can handle the initial list.
ruleset = [array('L', rule) for rule in new_section.values()]
# NOTE: gil must be held throughout this call
error = self.cfirewall.update_ruleset(i, ruleset)
if (error):
pass # TODO: do something here
self._initialize.done()
@cfg_read_poller('firewall_system', folder='iptables')
def monitor_system_rules(self, system_rules):
# 0-9: reserved - dns, dhcp, loopback, etc
# 10-159: zone mgmt rules. tens place designates interface index
# - 0: webui, 1: cli, 2: ssh, 3: ping
# 160+: system control (proxy bypass prevention)
# dnxfirewall services access (all local network interfaces). dhcp, dns, icmp, etc.
# DHCP discover/request allow
shell(
f'iptables -A INPUT -m mark ! --mark {WAN_IN} -p udp --dport 67 -j ACCEPT'
)
# implicit DNS allow for local users
shell(
f'iptables -A INPUT -m mark ! --mark {WAN_IN} -p udp --dport 53 -j ACCEPT'
)
# implicit http/s allow to dnx-web for local LAN users
shell(
f'iptables -A INPUT -m mark --mark {LAN_IN} -p tcp --dport 443 -j ACCEPT'
)
shell(
f'iptables -A INPUT -m mark --mark {LAN_IN} -p tcp --dport 80 -j ACCEPT'
)
# NOTE: these are default settings of user defined options. these can be removed from the webui after setup and are only
# here for convenience.
# dnxfirewall LAN interface ping allow
shell(
f'iptables -A MGMT -m mark --mark {LAN_IN} -p icmp -m icmp --icmp-type 8 -j ACCEPT'
)
# DMZ webui access
shell(
f'iptables -A MGMT -m mark --mark {DMZ_IN} -p tcp --dport 443 -j ACCEPT'
)
shell(
f'iptables -A MGMT -m mark --mark {DMZ_IN} -p tcp --dport 80 -j ACCEPT'
)
ruleset = load_configuration(system_rules,
filepath='dnx_system/iptables')
# sorting merged dict (system + usr), then taking values to convert into python arrays
ruleset = [array('L', rule) for rule in dict(sorted(ruleset)).values()]
# NOTE: gil must be held throughout this call
error = self.cfirewall.update_ruleset(0, ruleset)
if (error):
pass # TODO: do something here
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
class Configuration:
_setup = False
__slots__ = (
'initialize',
'IPS',
'_cfg_change',
)
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
@classmethod
def setup(cls, IPS):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(IPS.__name__)
self.IPS = IPS
self._load_passive_blocking()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_open_ports).start()
threading.Thread(target=self._update_system_vars).start()
self.initialize.wait_for_threads(count=3)
threading.Thread(target=self._clear_ip_tables).start()
# this resets any passively blocked hosts in the system on startup. persisting this
# data through service or system restarts is not really worth the energy.
def _load_passive_blocking(self):
self.IPS.fw_rules = dict(System.ips_passively_blocked())
@cfg_read_poller('ips')
def _get_settings(self, cfg_file):
ips = load_configuration(cfg_file)
self.IPS.ids_mode = ips['ids_mode']
self.IPS.ddos_prevention = ips['ddos']['enabled']
# ddos CPS configured thresholds
self.IPS.connection_limits = {
PROTO.ICMP: ips['ddos']['limits']['source']['icmp'],
PROTO.TCP: ips['ddos']['limits']['source']['tcp'],
PROTO.UDP: ips['ddos']['limits']['source']['udp']
}
self.IPS.portscan_prevention = ips['port_scan']['enabled']
self.IPS.portscan_reject = ips['port_scan']['reject']
if (self.IPS.ddos_prevention and not self.IPS.ids_mode):
# checking length(hours) to leave IP table rules in place for hosts part of ddos attacks
self.IPS.block_length = ips['passive_block_ttl'] * ONE_HOUR
# NOTE: this will provide a simple way to ensure very recently blocked hosts do not get their
# rule removed if passive blocking is disabled.
if (not self.IPS.block_length):
self.IPS.block_length = FIVE_MIN
# if ddos engine is disabled
else:
self.IPS.block_length = NO_DELAY
# src ips that will not trigger ips
self.IPS.ip_whitelist = set(
[IPv4Address(ip) for ip in ips['whitelist']['ip_whitelist']])
self._cfg_change.set()
self.initialize.done()
# NOTE: determine whether default sleep timer is acceptable for this method. if not, figure out how to override
# the setting set in the decorator or remove the decorator entirely.
@cfg_read_poller('ips')
def _get_open_ports(self, cfg_file):
ips = load_configuration(cfg_file)
self.IPS.open_ports = {
PROTO.TCP: {
int(local_port): int(wan_port)
for wan_port, local_port in ips['open_protocols']
['tcp'].items()
},
PROTO.UDP: {
int(local_port): int(wan_port)
for wan_port, local_port in ips['open_protocols']
['udp'].items()
}
}
self._cfg_change.set()
self.initialize.done()
@looper(NO_DELAY)
def _update_system_vars(self):
# waiting for any thread to report a change in configuration.
self._cfg_change.wait()
# resetting the config change event.
self._cfg_change.clear()
open_ports = self.IPS.open_ports[PROTO.TCP] or self.IPS.open_ports[
PROTO.UDP]
self.IPS.ps_engine_enabled = True if self.IPS.portscan_prevention and open_ports else False
self.IPS.ddos_engine_enabled = True if self.IPS.ddos_prevention else False
# makes some conditions easier when determining what to do with the packet.
self.IPS.all_engines_enabled = self.IPS.ps_engine_enabled and self.IPS.ddos_engine_enabled
self.initialize.done()
@looper(FIVE_MIN)
# NOTE: refactored function utilizing iptables + timestamp comment to identify rules to be expired.
# this should inherently make the passive blocking system persist service or system reboots.
# TODO: consider using the fw_rule dict check before continuing to call System.
def _clear_ip_tables(self):
expired_hosts = System.ips_passively_blocked(
block_length=self.IPS.block_length)
if (not expired_hosts):
return
with IPTablesManager() as iptables:
for host, timestamp in expired_hosts:
iptables.proxy_del_rule(host,
timestamp,
table='raw',
chain='IPS')
# removing host from ips tracker/ suppression dictionary
self.IPS.fw_rules.pop(IPv4Address(host),
None) # should never return None
def __init__(self, name):
self.Initialize = Initialize(Log, name)
class LanRestrict:
'''lan restriction management is done within this class.
public attributes: is_enabled, is_active
call run method to start service.
'''
_enabled = False
_active = False
__slots__ = ('IPProxy', 'initialize')
def __init__(self, name):
self.initialize = Initialize(Log, name)
@classproperty
def is_enabled(cls): # pylint: disable=no-self-argument
return cls._enabled
@classproperty
def is_active(cls): # pylint: disable=no-self-argument
return cls._active
@classmethod
def run(cls, IPProxy):
'''initializes settings and attributes then runs timer service in a new thread before returning.'''
self = cls(IPProxy.__name__)
self.IPProxy = IPProxy
cls.__load_status()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._tracker).start()
self.initialize.wait_for_threads(count=2)
@cfg_read_poller('ip_proxy')
def _get_settings(self, cfg_file):
ip_proxy = load_configuration(cfg_file)
enabled = ip_proxy['time_restriction']['enabled']
self._change_attribute('_enabled', enabled)
self.initialize.done()
@looper(ONE_MIN)
def _tracker(self):
restriction_start, restriction_end, now = self._calculate_times()
# Log.debug(f'ENABLED: {self.is_enabled} | ACTIVE: {self.is_active}')
# Log.debug(f'START: {restriction_start}: {datetime.fromtimestamp(restriction_start)}')
# Log.debug(f'NOW: {now}: {datetime.fromtimestamp(now)}')
# Log.debug(f'END: {restriction_end}: {datetime.fromtimestamp(restriction_end)}')
if (not self.is_enabled and self.is_active):
self._set_restriction_status(active=False)
# NOTE: validate end check is doing anything. if not remove it to make code easier to deal with
elif (self.is_enabled and not self.is_active
and restriction_start < now < restriction_end):
self._set_restriction_status(active=True)
Log.notice('LAN restriction in effect.')
elif (self.is_active and now > restriction_end):
self._set_restriction_status(active=False)
Log.notice('LAN restriction released.')
self.initialize.done()
# Calculating what the current date and time is and what the current days start time is in epoch
# this must be calculated daily as the start time epoch is always changing
def _calculate_times(self):
restriction_start, restriction_length, offset = self._load_restriction(
)
now = fast_time() + offset
c_d = [int(i) for i in System.date(now)] # current date
r_start = [int(i) for i in restriction_start.split(':')]
restriction_start = datetime(c_d[0], c_d[1], c_d[2], r_start[0],
r_start[1]).timestamp()
restriction_end = restriction_start + restriction_length
if (self.is_active):
restriction_end = load_configuration('ip_proxy_timer')['end']
else:
self._write_end_time(restriction_end)
return restriction_start, restriction_end, now
# Calculating the time.time() of when timer should end. calculated by current days start time (time since epoch)
# and then adding seconds of user configured amount to start time.
def _write_end_time(self, restriction_end):
with ConfigurationManager('ip_proxy_timer') as dnx:
time_restriction = dnx.load_configuration()
time_restriction['end'] = restriction_end
dnx.write_configuration(time_restriction)
def _load_restriction(self):
ip_proxy = load_configuration('ip_proxy')
logging = load_configuration('logging_client')
restriction_start = ip_proxy['time_restriction']['start']
restriction_length = ip_proxy['time_restriction']['length']
os_direction = logging['time_offset']['direction']
os_amount = logging['time_offset']['amount']
offset = int(f'{os_direction}{os_amount}') * ONE_DAY
return restriction_start, restriction_length, offset
def _set_restriction_status(self, active):
self._change_attribute('_active', active)
with ConfigurationManager('ip_proxy_timer') as dnx:
time_restriction = dnx.load_configuration()
time_restriction['active'] = active
dnx.write_configuration(time_restriction)
@classmethod
def __load_status(cls):
time_restriction = load_configuration('ip_proxy_timer')
cls._active = time_restriction['active']
@classmethod
def _change_attribute(cls, name, status):
setattr(cls, name, status)
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
@classmethod
def setup(cls, DHCPServer):
if (cls._setup):
raise RuntimeError('configuration setup should only be called once.')
cls._setup = True
self = cls(DHCPServer.__name__)
self.DHCPServer = DHCPServer
self._load_interfaces()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_server_options).start()
threading.Thread(target=self._get_reservations).start()
self.initialize.wait_for_threads(count=3)
@cfg_read_poller('dhcp_server')
def _get_settings(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)
# updating user configuration items per interface in memory.
for settings in dhcp_settings['interfaces'].values():
# NOTE ex. ident: eth0, lo, enp0s3
intf_identity = settings['ident']
enabled = True if settings['enabled'] else False
# TODO: compare interface status in memory with what is loaded in. if it is different then the setting was just
# changed and needs to be acted on. implement register/unregister methods available to external callers and use
# them to act on the disable of an interfaces dhcp service. this should also be the most efficient in that if
# all listeners are disabled only the automate class will be actively processing on file changes.
# NOTE: .get is to cover server startup. do not change. test functionality.
sock_fd = self.DHCPServer.intf_settings[intf_identity]['fileno']
if (enabled and not self.DHCPServer.intf_settings[intf_identity].get('enabled', False)):
self.DHCPServer.enable(sock_fd, intf_identity)
elif (not enabled and self.DHCPServer.intf_settings[intf_identity].get('enabled', True)):
self.DHCPServer.disable(sock_fd, intf_identity)
# identity will be kept in settings just in case, though they key is the identity also.
self.DHCPServer.intf_settings[intf_identity].update(settings)
self.initialize.done()
@cfg_read_poller('dhcp_server')
def _get_server_options(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)
server_options = dhcp_settings['options']
interfaces = dhcp_settings['interfaces']
# if server options have not changed, the function can return
if (server_options == self.DHCPServer.options): return
# will wait for 2 threads to check in before running code. this will allow the necessary settings
# to be initialized on startup before this thread continues.
self.initialize.wait_in_line(wait_for=2)
with self.DHCPServer.options_lock:
# iterating over server interfaces and populated server option data sets NOTE: consider merging server
# options with the interface settings since they are technically bound.
for intf, settings in self.DHCPServer.intf_settings.items():
for _intf in interfaces.values():
# ensuring the interfaces match since we cannot guarantee order
if (intf != _intf['ident']): continue
# converting keys to integers (json keys are string only), then packing any
# option value that is in ip address form to raw bytes.
for o_id, values in server_options.items():
opt_len, opt_val = values
if (not isinstance(opt_val, str)):
self.DHCPServer.options[intf][int(o_id)] = (opt_len, opt_val)
else:
# NOTE: this is temporary to allow interface netmask to be populated correction while migrating
# to new system backend functions.
if (o_id == '1'):
ip_value = get_netmask(interface=intf)
else:
ip_value = list(settings['ip'].network)[int(opt_val)]
# using digit as ipv4 network object index to grab correct ip object, then pack.
self.DHCPServer.options[intf][int(o_id)] = (
opt_len, ip_value.packed
)
self.initialize.done()
# loading user configured dhcp reservations from json config file into memory.
@cfg_read_poller('dhcp_server')
def _get_reservations(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)
# dict comp that retains all info of stored json data, but converts ip address into objects
self.DHCPServer.reservations = {
mac: {
'ip_address': IPv4Address(info['ip_address']),
'description': info['description']
}
for mac, info in dhcp_settings['reservations'].items()
}
# creating local reference for iteration performance
reservations = self.DHCPServer.reservations
# loaded all reserved ip addressing into a set to be referenced below
reserved_ips = set([IPv4Address(info['ip_address']) for info in reservations.values()])
# sets reserved ip addresses lease records to available is there are no longer configured
dhcp_leases = self.DHCPServer.leases
for ip, record in dhcp_leases.items():
# record[0] is record type. cross referencing ip reservation list with current lease table
# to reset any leased record placeholders for the reserved ip.
if (record[0] is DHCP.RESERVATION and IPv4Address(ip) not in reserved_ips):
dhcp_leases[ip] = _NULL_LEASE
# adding dhcp reservations to lease table to prevent them from being selected during an offer
self.DHCPServer.leases.update({
IPv4Address(info['ip_address']): (DHCP.RESERVATION, 0, mac) for mac, info in reservations.items()
})
self.initialize.done()
# accessing class object via local instance to change overall DHCP server enabled ints tuple
def _load_interfaces(self):
fw_intf = load_configuration('config')['interfaces']['builtins']
dhcp_intfs = load_configuration('dhcp_server')['interfaces']
# interface ident eg. eth0
for *_, intf in self.DHCPServer._intfs:
# interface friendly name eg. wan
for _intf, settings in dhcp_intfs.items():
# ensuring the interfaces match since we cannot guarantee order
if (intf != settings['ident']): continue
# creating ipv4 interface object which will be associated with the ident in the config.
# this can then be used by the server to identify itself as well as generate its effective
# subnet based on netmask for ip handouts or membership tests.
intf_ip = IPv4Interface(str(fw_intf[_intf]['ip']) + '/' + str(fw_intf[_intf]['netmask']))
# initializing server options so the auto loader doesnt have to worry about it.
self.DHCPServer.options[intf] = {}
# updating general network information for interfaces on server class object. these will never change
# while the server is running. for interfaces changes, the server must be restarted.
# initializing fileno key in the intf dict to make assignments easier in later calls.
self.DHCPServer.intf_settings[intf] = {'ip': intf_ip}
self._create_socket(intf)
Log.debug(f'loaded interfaces from file: {self.DHCPServer.intf_settings}')
# this is providing the first portion of creating a socket. this will allow the system to create the socket
# store the file descriptor id, and then bind when ready per normal registration logic.
def _create_socket(self, intf):
l_sock = socket(AF_INET, SOCK_DGRAM)
# used for converting interface identity to socket object file descriptor number
self.DHCPServer.intf_settings[intf].update({
'l_sock': l_sock,
'fileno': l_sock.fileno()
})
Log.debug(f'[{l_sock.fileno()}][{intf}] socket created')
def __init__(self):
self._initialize = Initialize(Log, 'LogService')
class LogService:
_log_modules = [
x for x in os.listdir(f'{HOME_DIR}/dnx_system/log')
if x not in EXCLUDED_MODULES
]
__slots__ = ('log_length', 'log_level', '_initialize')
def __init__(self):
self._initialize = Initialize(Log, 'LogService')
@classmethod
def run(cls):
self = cls()
threading.Thread(target=self.get_settings).start()
self._initialize.wait_for_threads(count=1)
threading.Thread(target=self.organize).start()
threading.Thread(target=self.clean_db_tables).start()
threading.Thread(target=self.clean_blocked_table).start()
# Recurring logic to gather all log files and add the mto a signle file (combined logs) every 5 minutes
@looper(THREE_MIN)
def organize(self):
# print('[+] Starting organize operation.')
log_entries = []
date = str_join(System.date())
for module in self._log_modules:
module_entries = self._combine_logs(module, date)
if (module_entries):
log_entries.extend(module_entries)
sorted_log_entries = sorted(log_entries)
if (sorted_log_entries):
self._write_combined_logs(sorted_log_entries, date)
del log_entries # to reclaim system memory
# grabbing the log from the sent in module, splitting the lines, and returning a list
# TODO: see if we can load file as generator
@staticmethod
def _combine_logs(module, date):
file_entries = []
if not os.path.isfile(
f'{HOME_DIR}/dnx_system/log/{module}/{date}-{module}.log'):
return None
with open(f'{HOME_DIR}/dnx_system/log/{module}/{date}-{module}.log',
'r') as log_file:
for _ in range(20):
line = log_file.readline().strip()
if not line: break
file_entries.append(line)
return file_entries
# writing the log entries to the combined log
@staticmethod
def _write_combined_logs(sorted_log_entries, date):
with open(f'{HOME_DIR}/dnx_system/log/combined/{date}-combined.log',
'w+') as system_log:
# print(f'writing {HOME_DIR}/dnx_system/log/combined/{date[0]}{date[1]}{date[2]}-combined.log')
for log in sorted_log_entries:
system_log.write(f'{log}\n')
@looper(ONE_DAY)
def clean_db_tables(self):
# print('[+] Starting general DB table cleaner.')
with DBConnector(Log) as FirewallDB:
for table in ['dnsproxy', 'ipproxy', 'ips', 'infectedclients']:
FirewallDB.table_cleaner(self.log_length, table=table)
# NOTE: consider moving this into the DBConnector so it can report if no exc are raised.
Log.notice('completed daily database cleaning')
@looper(THREE_MIN)
def clean_blocked_table(self):
# print('[+] Starting DB blocked table cleaner.')
with DBConnector(Log) as FirewallDB:
FirewallDB.blocked_cleaner(table='blocked')
# NOTE: consider moving this into the DBConnector so it can report if no exc are raised.
Log.debug('completed blocked database cleaning')
@cfg_read_poller('logging_client')
def get_settings(self, cfg_file):
# print('[+] Starting settings update poller.')
log_settings = load_configuration(cfg_file)
self.log_length = log_settings['logging']['length']
self.log_level = log_settings['logging']['level']
self._initialize.done()