class Configuration:
_service_setup = False
def __init__(self, name):
self.Initialize = Initialize(Log, name)
@classmethod
def service_setup(cls, SyslogService):
'''start threads for tasks required by the syslog service. blocking until settings are loaded/initialized.'''
if (cls._service_setup):
raise RuntimeError('service setup should only be called once.')
cls._service_setup = True
self = cls(SyslogService.__name__)
self.SyslogService = SyslogService
self.Initialize.wait_for_threads(count=1)
threading.Thread(target=self.get_settings).start()
@cfg_read_poller('syslog_client')
def get_settings(self, cfg_file):
syslog = load_configuration(cfg_file)['syslog']
SyslogService = self.SyslogService
SyslogService.syslog_enabled = syslog['enabled']
SyslogService.syslog_protocol = syslog['protocol']
SyslogService.tls_enabled = syslog['tls']['enabled']
SyslogService.self_signed_cert = syslog['tls']['self_signed']
SyslogService.tcp_fallback = syslog['tcp']['fallback']
SyslogService.udp_fallback = syslog['udp']['fallback']
syslog_servers = syslog['servers']
# if service is started without servers configured we will return here.
if not syslog_servers: return
names = ['primary', 'secondary']
with SyslogService.server_lock:
for name, cfg_server, mem_server in zip(names, syslog_servers.values(), SyslogService.syslog_servers):
if (cfg_server['ip_address'] == mem_server.get('ip')): continue
getattr(SyslogService.syslog_servers, name).update({
'ip': syslog_servers[name]['ip_address'],
PROTO.UDP: True, PROTO.TCP: True, PROTO.DNS_TLS: True
})
def get_interface_settings(self):
interface_settings = load_configuration('config.json')
self.lan_int = interface_settings['settings']['interface']['inside']
def __init__(self, name):
self.initialize = Initialize(Log, name)
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
@classmethod
def setup(cls, DHCPServer):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(DHCPServer.__name__)
self.DHCPServer = DHCPServer
self._load_interfaces()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_server_options).start()
threading.Thread(target=self._get_reservations).start()
self.initialize.wait_for_threads(count=3)
@cfg_read_poller('dhcp_server')
def _get_settings(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)['dhcp_server']
# TODO: add a change detection check??
# updating user configuration items per interface in memory.
for settings in dhcp_settings['interfaces'].values():
# NOTE: probably temporary. same as below
if not settings['enabled']: continue
# NOTE ex. ident: eth0, lo, enp0s3
intf_identity = settings['ident']
# identity will be kept in settings just in case, though they key is the identity also.
self.DHCPServer.intf_settings[intf_identity].update(settings)
self.initialize.done()
@cfg_read_poller('dhcp_server')
def _get_server_options(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)['dhcp_server']
server_options = dhcp_settings['options']
interfaces = dhcp_settings['interfaces']
# if server options have not changed, the function can return
if (server_options == self.DHCPServer.options): return
# will wait for 2 threads to checking before running code. this will allow the necessary settings
# to be initialized on startup before this thread continues.
self.initialize.wait_in_line(2)
with self.DHCPServer.options_lock:
# iterating over server interfaces and populated server option data sets NOTE: consider merging server
# options with the interface settings since they are technically bound.
for intf, settings in self.DHCPServer.intf_settings.items():
for _intf in interfaces.values():
# ensuring the iterfaces match since we cannot guarantee order
if (intf != _intf['ident']): continue
# NOTE: should be temporary, while dmz is not fully implemented
if not settings['enabled']: continue
# converting keys to integers (json keys are string only), then packing any
# option value that is in ip address form to raw bytes.
for o_id, values in server_options.items():
opt_len, opt_val = values
if (not isinstance(opt_val, str)):
self.DHCPServer.options[intf][int(o_id)] = (
opt_len, opt_val)
else:
# NOTE: this is temporary to allow interface netmask to be populated correction while migrating
# to new system backend functions.
if (o_id == '1'):
ip_value = get_netmask(interface=intf)
else:
ip_value = list(
settings['ip'].network)[int(opt_val)]
# using digit as ipv4 network object index to grab correct ip object, then pack.
self.DHCPServer.options[intf][int(o_id)] = (
opt_len, ip_value.packed)
self.initialize.done()
# loading user configured dhcp reservations from json config file into memory.
@cfg_read_poller('dhcp_server')
def _get_reservations(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)['dhcp_server']
self.DHCPServer.reservations = dhcp_settings['reservations']
reservations = self.DHCPServer.reservations
reservation_list = list(reservations.items())
dhcp_leases = list(self.DHCPServer.leases.items())
for ip, record in dhcp_leases:
r_type, *_ = record
ip_object = IPv4Address(ip)
if (r_type is DHCP.RESERVATION and ip_object not in reservations):
self.DHCPServer.leases.pop(ip_object)
# configuring dhcp reservations
self.DHCPServer.leases.update({
IPv4Address(ip): (DHCP.RESERVATION, 0, mac)
for ip, mac in reservation_list
})
self.initialize.done()
# accessing class object via local instance to change overall DHCP server enabled ints tuple
def _load_interfaces(self):
fw_settings = load_configuration('config')['settings']
server_settings = load_configuration('dhcp_server')['dhcp_server']
fw_intf = fw_settings['interfaces']
dhcp_intfs = server_settings['interfaces']
# ident
for intf in self.DHCPServer._intfs:
# friendly name
for _intf, settings in dhcp_intfs.items():
# ensuring the iterfaces match since we cannot guarantee order
if (intf != settings['ident']): continue
# passing over disabled server interfaces. NOTE: DEF temporary
if not dhcp_intfs[_intf]['enabled']: continue
# creating ipv4 interface object which will be associated with the ident in the config.
# this can then be used by the server to identify itself as well as generate its effective
# subnet based on netmask for ip handouts or membership tests.
intf_ip = IPv4Interface(
str(fw_intf[_intf]['ip']) + '/' +
str(fw_intf[_intf]['netmask']))
# initializing server options so the auto loader doesnt have to worry about it.
self.DHCPServer.options[intf] = {}
# updating general network information for interfaces on server class object. these will never change
# while the server is running. for interfaces changes, the server must be restarted.
self.DHCPServer.intf_settings[intf] = {'ip': intf_ip}
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
@classmethod
def setup(cls, IPS):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(IPS.__name__)
self.IPS = IPS
self._load_interfaces()
self._manage_ip_tables()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_open_ports).start()
threading.Thread(target=self._update_system_vars).start()
self.initialize.wait_for_threads(count=3)
threading.Thread(target=self._clear_ip_tables).start()
def _manage_ip_tables(self):
IPTableManager.purge_proxy_rules(table='mangle', chain='IPS')
def _load_interfaces(self):
dnx_settings = load_configuration('config')['settings']
wan_ident = dnx_settings['interfaces']['wan']['ident']
self.IPS.broadcast = Interface.broadcast_address(wan_ident)
@cfg_read_poller('ips')
def _get_settings(self, cfg_file):
ips = load_configuration(cfg_file)['ips']
self.IPS.ids_mode = ips['ids_mode']
self.IPS.ddos_prevention = ips['ddos']['enabled']
# ddos CPS configured thresholds
self.IPS.connection_limits = {
PROTO.ICMP: ips['ddos']['limits']['source']['icmp'],
PROTO.TCP: ips['ddos']['limits']['source']['tcp'],
PROTO.UDP: ips['ddos']['limits']['source']['udp']
}
self.IPS.portscan_prevention = ips['port_scan']['enabled']
self.IPS.portscan_reject = ips['port_scan']['reject']
# checking length(hours) to leave IP Table Rules in place for hosts part of ddos attacks
if (self.IPS.ddos_prevention and not self.IPS.ids_mode):
# NOTE: this will provide a simple way to ensure very recently blocked hosts do not get their
# rule removed if passive blocking is disabled.
if (not self.IPS.block_length):
self.IPS.block_length = FIVE_MIN
else:
self.IPS.block_length = ips['passive_block_ttl'] * ONE_HOUR
# if ddos engine is disabled
else:
self.IPS.block_length = 0
# src ips that will not trigger ips
self.IPS.ip_whitelist = set(
[IPv4Address(ip) for ip in ips['whitelist']['ip_whitelist']])
self._cfg_change.set()
self.initialize.done()
# NOTE: determine whether default sleep timer is acceptible for this method. if not, figure out how to override
# the setting set in the decorator or remove the decorator entirely.
@cfg_read_poller('ips')
def _get_open_ports(self, cfg_file):
ips = load_configuration(cfg_file)['ips']
self.IPS.open_ports = {
PROTO.TCP: {
int(local_port): int(wan_port)
for wan_port, local_port in ips['open_protocols']
['tcp'].items()
},
PROTO.UDP: {
int(local_port): int(wan_port)
for wan_port, local_port in ips['open_protocols']
['udp'].items()
}
}
self._cfg_change.set()
self.initialize.done()
@looper(NO_DELAY)
def _update_system_vars(self):
# waiting for any thread to report a change in configuration.
self._cfg_change.wait()
#resetting the config change event.
self._cfg_change.clear()
open_ports = self.IPS.open_ports[PROTO.TCP] or self.IPS.open_ports[
PROTO.UDP]
if (self.IPS.ddos_prevention
or (self.IPS.portscan_prevention and open_ports)):
self.IPS.ins_engine_enabled = True
else:
self.IPS.ins_engine_enabled = False
if (self.IPS.portscan_prevention and open_ports):
self.IPS.ps_engine_enabled = True
else:
self.IPS.ps_engine_enabled = False
if (self.IPS.ddos_prevention):
self.IPS.ddos_engine_enabled = True
else:
self.IPS.ddos_engine_enabled = False
self.initialize.done()
@looper(THIRTY_MIN)
# TODO: consider making this work off of a thread event. then we can convert the dynamic looper
# to a standard looper and method will block until an actual host has been blocked.
def _clear_ip_tables(self):
# quick check to see if any firewall rules exist
if not self.IPS.firewall_rules: return
firewall_rules = self.IPS.firewall_rules
block_length = self.IPS.block_length
now = fast_time()
with IPTableManager() as iptables:
for tracked_ip, insert_time in list(firewall_rules.items()):
if (now - insert_time > block_length) and firewall_rules.pop(
tracked_ip, None):
iptables.proxy_del_rule(tracked_ip,
table='mangle',
chain='IPS')
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
@classmethod
def setup(cls, IPProxy):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(IPProxy.__name__)
self.IPProxy = IPProxy
self._load_interfaces()
self._manage_ip_tables()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_ip_whitelist).start()
threading.Thread(target=self._get_open_ports).start()
self.initialize.wait_for_threads(count=3)
def _load_interfaces(self):
dnx_settings = load_configuration('config')['settings']
lan_net = dnx_settings['interfaces']['lan']['subnet']
self.IPProxy.lan_net = IPv4Network(lan_net)
@cfg_read_poller('ip_proxy')
def _get_settings(self, cfg_file):
ip_proxy = load_configuration(cfg_file)['ip_proxy']
cat_settings = ip_proxy['categories']
geo_settings = ip_proxy['geolocation']
cat_enabled = []
for cat, setting in cat_settings.items():
if (setting): cat_enabled.append(1)
self.IPProxy.cat_settings[IPP_CAT[cat.upper()]] = DIR(setting)
geo_enabled = []
for cat, setting in geo_settings.items():
if (setting): geo_enabled.append(1)
# using enum for category key and direction value
try:
self.IPProxy.geo_settings[GEO[cat.title()]] = DIR(setting)
except KeyError:
continue # NOTE: temporary while not all enums/countries are populated
self.IPProxy.inspect_on = bool(cat_enabled or geo_enabled)
self.IPProxy.cat_enabled = bool(cat_enabled)
self.IPProxy.geo_enabled = bool(geo_enabled)
self.IPProxy.ids_mode = ip_proxy['ids_mode']
self.initialize.done()
@cfg_read_poller('whitelist')
def _get_ip_whitelist(self, cfg_file):
whitelist = load_configuration(cfg_file)['whitelist']
whitelist = whitelist['ip_whitelist']
self.IPProxy.ip_whitelist = {
ip
for ip, wl_info in whitelist.items() if wl_info['type'] == 'ip'
}
self.IPProxy.tor_whitelist = {
ip
for ip, wl_info in whitelist.items() if wl_info['type'] == 'tor'
}
self.initialize.done()
@cfg_read_poller('ips')
def _get_open_ports(self, cfg_file):
ips = load_configuration(cfg_file)['ips']
open_tcp_ports = ips['open_protocols']['tcp']
open_udp_ports = ips['open_protocols']['udp']
self.IPProxy.open_ports = {
PROTO.TCP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_tcp_ports.items()
},
PROTO.UDP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_udp_ports.items()
}
}
self.initialize.done()
def _manage_ip_tables(self):
IPTableManager.clear_dns_over_https()
IPTableManager.update_dns_over_https()
@staticmethod
# Loading lists of interesting traffic into dictionaries and creating ip table rules for dns over https blocking
def load_ip_signature_bitmaps():
list_files = ListFiles(Log=Log)
list_files.combine_ips()
list_files.combine_geolocation()
ip_category_signatures = load_ip_bitmap(Log)
geolocation_signatures = load_geo_bitmap(Log)
return ip_category_signatures, geolocation_signatures
class LanRestrict:
'''lan restriction management is done within this class.
public attributes: is_enabled, is_active
call run method to start service.
'''
_enabled = False
_active = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
@classproperty
def is_enabled(cls): # pylint: disable=no-self-argument
return cls._enabled
@classproperty
def is_active(cls): # pylint: disable=no-self-argument
return cls._active
@classmethod
def run(cls, IPProxy):
'''initializes settings and attributes then runs timer service in a new thread before returning.'''
self = cls(IPProxy.__name__)
self.IPProxy = IPProxy
cls.__load_status()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._tracker).start()
self.initialize.wait_for_threads(count=2)
@cfg_read_poller('ip_proxy')
def _get_settings(self, cfg_file):
restriction_settings = load_configuration(cfg_file)
enabled = restriction_settings['ip_proxy']['time_restriction'][
'enabled']
self._change_attribute('_enabled', enabled)
self.initialize.done()
@looper(ONE_MIN)
def _tracker(self):
restriction_start, restriction_end, now = self._calculate_times()
Log.debug(f'ENABLED: {self.is_enabled} | ACTIVE: {self.is_active}')
Log.debug(
f'START: {restriction_start}: {datetime.fromtimestamp(restriction_start)}'
)
Log.debug(f'NOW: {now}: {datetime.fromtimestamp(now)}')
Log.debug(
f'END: {restriction_end}: {datetime.fromtimestamp(restriction_end)}'
)
if (not self.is_enabled and self.is_active):
self._set_restriction_status(active=False)
# NOTE: validate end check is doing anything. if not remove it to make code easier to deal with
elif (self.is_enabled and not self.is_active
and restriction_start < now < restriction_end):
self._set_restriction_status(active=True)
Log.notice('LAN Time Restriction in effect.')
elif (self.is_active and now > restriction_end):
self._set_restriction_status(active=False)
Log.notice('LAN Time Restriction released.')
self.initialize.done()
# Calculating what the current date and time is and what the current days start time is in epoch
# this must be calculated daily as the start time epoch is always changing
def _calculate_times(self):
restriction_start, restriction_length, offset = self._load_restriction(
)
now = time.time() + offset
c_d = [int(i) for i in System.date(now)] # current date
restriction_start = restriction_start.split(':')
restriction_start = [int(i) for i in restriction_start]
restriction_start = datetime(c_d[0], c_d[1], c_d[2],
restriction_start[0],
restriction_start[1]).timestamp()
restriction_start = restriction_start
restriction_end = restriction_start + restriction_length
if (self.is_active):
restriction_status = load_configuration('ip_proxy_timer.json')
restriction_end = restriction_status['time_restriction']['end']
else:
self._write_end_time(restriction_end)
return restriction_start, restriction_end, now
# Calculating the time.time() of when timer should end. calculated by current days start time (time since epoch)
# and then adding seconds of user configured amount to start time.
def _write_end_time(self, restriction_end):
with ConfigurationManager('ip_proxy_timer.json') as dnx:
time_restriction = dnx.load_configuration()
time_restriction['time_restriction'].update(
{'end': restriction_end})
dnx.write_configuration(time_restriction)
def _load_restriction(self):
restriction = load_configuration('ip_proxy.json')
offset_settings = load_configuration('logging_client.json')
restriction_start = restriction['ip_proxy']['time_restriction'][
'start']
restriction_length = restriction['ip_proxy']['time_restriction'][
'length']
offset = offset_settings['logging']['time_offset']
os_direction = offset['direction']
os_amount = offset['amount']
offset = int(f'{os_direction}{os_amount}') * 3600
return restriction_start, restriction_length, offset
def _set_restriction_status(self, active):
self._change_attribute('_active', active)
with ConfigurationManager('ip_proxy_timer') as dnx:
restriction_status = dnx.load_configuration()
restriction_status['time_restriction']['active'] = active
dnx.write_configuration(restriction_status)
@classmethod
def __load_status(cls):
time_restriction = load_configuration('ip_proxy_timer')
cls._active = time_restriction['time_restriction']['active']
@classmethod
def _change_attribute(cls, name, status):
setattr(cls, name, status)
class Configuration:
_proxy_setup = False
_server_setup = False
_keywords = []
__slots__ = (
# callbacks
'DNSProxy',
'DNSServer',
'DNSCache',
# protected vars
'_initialize',
)
def __init__(self, name):
self._initialize = Initialize(Log, name)
@classmethod
def proxy_setup(cls, DNSProxy):
'''start threads for tasks required by the DNS proxy. blocking until settings are loaded/initialized.'''
if (cls._proxy_setup):
raise RuntimeError('proxy setup should only be called once.')
cls._proxy_setup = True
# NOTE: might be temporary, but this needed to be moved outside of the standard/bitmap sigs since they are
# now being handled by an external C extension (cython)
cls._keywords = load_keywords(Log=Log)
self = cls(DNSProxy.__name__)
self.DNSProxy = DNSProxy
threading.Thread(target=self._get_proxy_settings).start()
threading.Thread(target=self._get_list, args=('whitelist', )).start()
threading.Thread(target=self._get_list, args=('blacklist', )).start()
self._initialize.wait_for_threads(count=3)
@classmethod
def server_setup(cls, DNSServer, DNSCache):
'''start threads for tasks required by the DNS server. This will ensure all automated threads
get started, including reachability. blocking until settings are loaded/initialized.'''
if (cls._server_setup):
raise RuntimeError('server setup should only be called once.')
cls._server_setup = True
self = cls(DNSServer.__name__)
self.DNSServer = DNSServer
self.DNSCache = DNSCache
threading.Thread(target=self._get_server_settings).start()
self._initialize.wait_for_threads(count=1)
@cfg_read_poller('dns_proxy')
def _get_proxy_settings(self, cfg_file):
dns_proxy = load_configuration(cfg_file)['dns_proxy']
signatures = self.DNSProxy.signatures
# CATEGORY SETTINGS
enabled_keywords = []
for cat, setting in dns_proxy['categories']['default'].items():
# identifying enabled keyword search categories
if (setting['keyword']):
enabled_keywords.append(DNS_CAT[cat])
# identifying enabled general categories
if (setting['enabled']):
signatures.en_dns.add(DNS_CAT[cat])
# removing category if present in memory
else:
dns_cat = DNS_CAT[cat]
if (dns_cat in signatures.en_dns):
signatures.en_dns.remove(dns_cat)
# KEYWORD SETTINGS
# copying keyword signature list in memory to a local object. iterating over list. if the current item
# category is not an enabled category the signature will get removed and offset will get adjustest to
# ensure the index stay correct.
mem_keywords, offset = signatures.keyword.copy(), 0
for i, signature in enumerate(mem_keywords):
_, cat = signature
if cat not in enabled_keywords:
signatures.keyword.pop(i - offset)
offset += 1
# iterating over keywords from the signature set. if the keyword category is enabled and the current
# signature is not already in memory it will be added.
for signature, cat in self._keywords:
if cat in enabled_keywords and signature not in signatures.keyword:
signatures.keyword.append((signature, cat))
# TLD SETTINGS | generator
for tld, setting in load_tlds():
signatures.tld[tld] = setting
self._initialize.done()
@cfg_read_poller('dns_server')
def _get_server_settings(self, cfg_file):
DNSServer = self.DNSServer
dns_server = load_configuration(cfg_file)['dns_server']
tls_enabled = dns_server['tls']['enabled']
DNSServer.protocol = PROTO.DNS_TLS if tls_enabled else PROTO.UDP
DNSServer.udp_fallback = dns_server['tls']['fallback']
names = ['primary', 'secondary']
dns_servers = dns_server['resolvers']
with DNSServer.server_lock:
for name, cfg_server, mem_server in zip(names,
dns_servers.values(),
DNSServer.dns_servers):
if (cfg_server['ip_address'] == mem_server.get('ip')): continue
getattr(DNSServer.dns_servers, name).update({
'ip':
dns_servers[name]['ip_address'],
PROTO.UDP:
True,
PROTO.DNS_TLS:
True
})
DNSServer.dns_records = dns_server['records']
# CLEAR DNS or TOP Domains cache
self.DNSCache.clear_dns_cache = dns_server['cache']['standard']
self.DNSCache.clear_top_domains = dns_server['cache']['top_domains']
self._initialize.done()
@cfg_write_poller
# handles updating user defined signatures in memory/propogated changes to disk.
def _get_list(self, lname, cfg_file, last_modified_time):
timeout_detected = self._check_for_timeout(lname)
memory_list = getattr(self.DNSProxy, lname).dns
# if a rule timeout is detected for an entry in memory. we will update the config file
# to align with active rules, then we will remove the rules from memory.
if (timeout_detected):
loaded_list = self._update_list_file(lname, cfg_file)
self._modify_memory(memory_list, loaded_list, action=CFG.DEL)
# if file has been modified the modified list will be referenced to make in place changes to memory
# list, specifically around adding new rules and the new modified time will be returned. if not modified,
# the last modified time is returned and not changes are made.
modified_time = os.stat(f'{HOME_DIR}/dnx_system/data/{cfg_file}')
if (modified_time == last_modified_time):
return last_modified_time
loaded_list = load_configuration(cfg_file)[lname]['domain']
self._modify_memory(memory_list, loaded_list, action=CFG.ADD)
# ip whitelist specific. will do an inplace swap of all rules needing to be added or removed the memory.
if (lname == 'whitelist'):
self._modify_ip_whitelist(cfg_file)
self._initialize.done()
return modified_time
def _modify_memory(self, memory_list, loaded_list, *, action):
'''removing/adding signature/rule from memory as needed.'''
if (action in [CFG.ADD, CFG.ADD_DEL]):
# iterating over rules/signatures pulled from file
for rule, settings in loaded_list.items():
bitmap_key = convert_string_to_bitmap(rule, DNS_BIN_OFFSET)
# adding rule/signature to memory if not present
if (bitmap_key not in memory_list):
settings['key'] = rule
memory_list[bitmap_key] = settings
if (action in [CFG.DEL, CFG.ADD_DEL]):
# iterating over rules/signature in memory
for rule, settings in memory_list.copy().items():
bitmap_key = convert_string_to_bitmap(rule, DNS_BIN_OFFSET)
# if rule is not present in config file it will be removed from memory
if (settings['key'] not in loaded_list):
memory_list.pop(rule)
def _modify_ip_whitelist(self, cfg_file):
memory_ip_list = self.DNSProxy.whitelist.ip
loaded_ip_list = load_configuration(
cfg_file)['whitelist']['ip_whitelist']
# iterating over ip rules in memory.
for ip in memory_ip_list.copy():
# if it is not in the config file it will be removed.
if (f'{ip}' not in loaded_ip_list):
memory_ip_list.pop(ip)
# iterating over ip rules in configuration file
for ip, settings in loaded_ip_list.items():
# convert to ip address object which is the type stored as key
ip = IPv4Address(ip)
# if it is not in memory and the rule type is "global" it will be added
if (ip not in memory_ip_list and settings['type'] == 'global'):
memory_ip_list[ip] = True
# checking corresponding list file for any time based rules timing out. will return True if timeout
# is detected otherwise return False.
def _check_for_timeout(self, lname):
now = fast_time()
for info in getattr(self.DNSProxy, lname).dns.values():
if (now < info['expire']): continue
return True
return False
# updating the file with necessary changes.
def _update_list_file(self, lname, cfg_file):
now = fast_time()
with ConfigurationManager(cfg_file) as dnx:
lists = dnx.load_configuration()
loaded_list = lists[lname]['domain']
for domain, info in loaded_list.copy().items():
if (now < info['expire']): continue
loaded_list.pop(domain, None)
dnx.write_configuration(lists)
return loaded_list
@staticmethod
def load_dns_signature_bitmap():
ListFile = ListFiles(Log=Log)
ListFile.combine_domains()
wl_exceptions = load_configuration(
'whitelist')['whitelist']['exception']
bl_exceptions = load_configuration(
'blacklist')['blacklist']['exception']
return load_dns_bitmap(Log, bl_exc=bl_exceptions, wl_exc=wl_exceptions)
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
@classmethod
def setup(cls, IPS):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(IPS.__name__)
self.IPS = IPS
self._load_interfaces()
self._manage_ip_tables()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_open_ports).start()
threading.Thread(target=self._update_system_vars).start()
self.initialize.wait_for_threads(count=3)
threading.Thread(target=self._clear_ip_tables).start()
def _manage_ip_tables(self):
IPTableManager.purge_proxy_rules(table='mangle', chain='IPS')
def _load_interfaces(self):
dnx_settings = load_configuration('config')['settings']
wan_ident = dnx_settings['interfaces']['wan']['ident']
self.IPS.broadcast = Interface.broadcast_address(wan_ident)
@cfg_read_poller('ips')
def _get_settings(self, cfg_file):
ips = load_configuration(cfg_file)['ips']
self.IPS.ids_mode = ips['ids_mode']
self.IPS.ddos_prevention = ips['ddos']['enabled']
# ddos CPS configured thresholds
tcp_src_limit = ips['ddos']['limits']['source']['tcp']
udp_src_limit = ips['ddos']['limits']['source']['udp']
icmp_src_limit = ips['ddos']['limits']['source']['icmp']
self.IPS.connection_limits = {
PROTO.ICMP: icmp_src_limit,
PROTO.TCP: tcp_src_limit,
PROTO.UDP: udp_src_limit
}
self.IPS.portscan_prevention = ips['port_scan']['enabled']
self.IPS.portscan_reject = ips['port_scan']['reject']
# checking length(hours) to leave IP Table Rules in place for hosts part of ddos attacks
self.IPS.block_length = 0
if (self.IPS.ddos_prevention and not self.IPS.ids_mode):
self.IPS.block_length = ips['passive_block_ttl'] * ONE_HOUR
# NOTE: this will provide a simple way to ensure very recently blocked hosts do not get their
# rule removed if passive blocking is disabled.
if (not self.IPS.block_length):
self.IPS.block_length = TEN_MIN
# src ips that will not trigger ips
self.IPS.ip_whitelist = set(
[IPv4Address(ip) for ip in ips['whitelist']['ip_whitelist']])
self._cfg_change.set()
self.initialize.done()
# NOTE: determine whether default sleep timer is acceptible for this method. if not, figure out how to override
# the setting set in the decorator or remove the decorator entirely.
@cfg_read_poller('ips')
def _get_open_ports(self, cfg_file):
ips = load_configuration(cfg_file)['ips']
open_tcp_ports = ips['open_protocols']['tcp']
open_udp_ports = ips['open_protocols']['udp']
self.IPS.open_ports = {
PROTO.TCP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_tcp_ports.items()
},
PROTO.UDP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_udp_ports.items()
}
}
self._cfg_change.set()
self.initialize.done()
@looper(NO_DELAY)
def _update_system_vars(self):
# waiting for any thread to report a change in configuration.
self._cfg_change.wait()
#resetting the config change event.
self._cfg_change.clear()
open_ports = self.IPS.open_ports[PROTO.TCP] or self.IPS.open_ports[
PROTO.UDP]
if (self.IPS.ddos_prevention
or (self.IPS.portscan_prevention and open_ports)):
self.IPS.ins_engine_enabled = True
else:
self.IPS.ins_engine_enabled = False
if (self.IPS.portscan_prevention and open_ports):
self.IPS.ps_engine_enabled = True
else:
self.IPS.ps_engine_enabled = False
if (self.IPS.ddos_prevention):
self.IPS.ddos_engine_enabled = True
else:
self.IPS.ddos_engine_enabled = False
self.initialize.done()
@dynamic_looper
def _clear_ip_tables(self):
IPS = self.IPS
if (not IPS.fw_rules): return ONE_MIN
now, ips_to_remove = fast_time(), []
for tracked_ip, insert_time in IPS.fw_rules.items():
if (now - insert_time > IPS.block_length):
ips_to_remove.append(tracked_ip)
if (not ips_to_remove): return FIVE_MIN
with IPTableManager() as iptables:
for tracked_ip in ips_to_remove:
# NOTE: if the system is configured in IDS mode when the rule was created, an active rule would not have
# be inserted into the system. this will still run, but effectively do nothing. maybe we can figure out
# a way to identify the type of rule in the data set. remember that there could be some thread safety
# concerns when dealing with this issue so make sure solution is well thought out.
if IPS.fw_rules.pop(tracked_ip, None):
iptables.proxy_del_rule(tracked_ip,
table='mangle',
chain='IPS')
return FIVE_MIN
class Configuration:
_setup = False
def __init__(self, name):
self.initialize = Initialize(Log, name)
self._cfg_change = threading.Event()
@classmethod
def setup(cls, IPS):
if (cls._setup):
raise RuntimeError(
'configuration setup should only be called once.')
cls._setup = True
self = cls(IPS.__name__)
self.IPS = IPS
self._load_interfaces()
self._manage_ip_tables()
threading.Thread(target=self._get_settings).start()
threading.Thread(target=self._get_open_ports).start()
threading.Thread(target=self._ip_whitelist).start()
threading.Thread(target=self._update_system_vars).start()
self.initialize.wait_for_threads(count=4)
threading.Thread(target=self._clear_ip_tables).start()
def _manage_ip_tables(self):
IPTableManager.purge_proxy_rules(table='mangle', chain='IPS')
def _load_interfaces(self):
interface = load_configuration('config.json')
self.IPS.wan_int = interface['settings']['interfaces']['wan']
self.IPS.broadcast = Interface.broadcast_address(
self.IPS.wan_int['ident'])
@cfg_read_poller('ips')
def _get_settings(self, cfg_file):
# print('[+] Starting: IPS Settings Update Thread.')
ips = load_configuration(cfg_file)['ips']
self.IPS.ddos_prevention = ips['ddos']['enabled']
self.IPS.portscan_prevention = ips['port_scan']['drop']
# ddos CPS THRESHHOLD CHECK
tcp_src_limit = ips['ddos']['limits']['source']['tcp']
udp_src_limit = ips['ddos']['limits']['source']['udp']
icmp_src_limit = ips['ddos']['limits']['source']['icmp']
self.IPS.connection_limits = {
PROTO.ICMP: icmp_src_limit,
PROTO.TCP: tcp_src_limit,
PROTO.UDP: udp_src_limit
}
##Checking length(hours) to leave IP Table Rules in place for hosts part of ddos attacks
self.IPS.block_length = 0
if (self.IPS.portscan_prevention or self.IPS.ddos_prevention):
self.IPS.block_length = ips['passive_block_ttl'] * 3600
## Reject packet (tcp reset and icmp port unreachable)
self.IPS.portscan_reject = ips['port_scan']['reject']
## whitelist configured dns servers (local instance var)
self.whitelist_dns_servers = ips['whitelist']['dns_servers']
self._cfg_change.set()
self.initialize.done()
# NOTE: determine whether default sleep timer is acceptible for this method. if not, figure out how to override
# the setting set in the decorator or remove the decorator entirely.
@cfg_read_poller('ips')
def _get_open_ports(self, cfg_file):
ips_settings = load_configuration(cfg_file)
ips = ips_settings['ips']
open_tcp_ports = ips['open_protocols']['tcp']
open_udp_ports = ips['open_protocols']['udp']
self.IPS.open_ports = {
PROTO.TCP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_tcp_ports.items()
},
PROTO.UDP: {
int(local_port): int(wan_port)
for wan_port, local_port in open_udp_ports.items()
}
}
self._cfg_change.set()
self.initialize.done()
@cfg_read_poller('ips')
def _ip_whitelist(self, cfg_file):
whitelist = load_configuration(cfg_file)
ip_whitelist = set(whitelist['ips']['whitelist']['ip_whitelist'])
if (self.whitelist_dns_servers):
dns_servers_settings = load_configuration('dns_server.json')
dns_servers = dns_servers_settings['dns_server']
dns1 = dns_servers['resolvers']['server1']['ip_address']
dns2 = dns_servers['resolvers']['server2']['ip_address']
self.IPS.ip_whitelist = ip_whitelist.union({dns1, dns2})
else:
self.IPS.ip_whitelist = ip_whitelist
self._cfg_change.set()
self.initialize.done()
@looper(NO_DELAY)
def _update_system_vars(self):
# waiting for any thread to report a change in configuration.
self._cfg_change.wait()
#resetting the config change event.
self._cfg_change.clear()
open_ports = bool(self.IPS.open_ports[PROTO.TCP]
or self.IPS.open_ports[PROTO.UDP])
if (self.IPS.ddos_prevention
or (self.IPS.portscan_prevention and open_ports)):
self.IPS.ins_engine_enabled = True
else:
self.IPS.ins_engine_enabled = False
if (self.IPS.portscan_prevention and open_ports):
self.IPS.ps_engine_enabled = True
else:
self.IPS.ps_engine_enabled = False
if (self.IPS.ddos_prevention):
self.IPS.ddos_engine_enabled = True
else:
self.IPS.ddos_engine_enabled = False
self.initialize.done()
@dynamic_looper
def _clear_ip_tables(self):
ips_to_remove = []
if (self.IPS.active_ddos or not self.IPS.ddos_prevention):
return ONE_MIN
now = time.time()
for tracked_ip, rule_info in self.IPS.fw_rules.items():
time_added = rule_info['timestamp']
if (now - time_added < self.IPS.block_length): continue
ips_to_remove.append(tracked_ip)
if (not ips_to_remove): return FIVE_MIN
with IPTableManager() as iptables:
for tracked_ip in ips_to_remove:
if not self.IPS.fw_rules.pop(tracked_ip, None): continue
iptables.proxy_del_rule(tracked_ip,
table='mangle',
chain='IPS')
return FIVE_MIN