def _load_interfaces(self):
fw_intf = load_configuration('config')['interfaces']['builtins']
dhcp_intfs = load_configuration('dhcp_server')['interfaces']
# interface ident eg. eth0
for *_, intf in self.DHCPServer._intfs:
# interface friendly name eg. wan
for _intf, settings in dhcp_intfs.items():
# ensuring the interfaces match since we cannot guarantee order
if (intf != settings['ident']): continue
# creating ipv4 interface object which will be associated with the ident in the config.
# this can then be used by the server to identify itself as well as generate its effective
# subnet based on netmask for ip handouts or membership tests.
intf_ip = IPv4Interface(str(fw_intf[_intf]['ip']) + '/' + str(fw_intf[_intf]['netmask']))
# initializing server options so the auto loader doesnt have to worry about it.
self.DHCPServer.options[intf] = {}
# updating general network information for interfaces on server class object. these will never change
# while the server is running. for interfaces changes, the server must be restarted.
# initializing fileno key in the intf dict to make assignments easier in later calls.
self.DHCPServer.intf_settings[intf] = {'ip': intf_ip}
self._create_socket(intf)
Log.debug(f'loaded interfaces from file: {self.DHCPServer.intf_settings}')
def dns_status():
dns_servers_status = load_configuration('dns_server_status')
dns_server = load_configuration('dns_server')
tls_enabled = dns_server['tls']['enabled']
dns_servers = dns_server['resolvers']
dns_servers_copy = deepcopy(dns_servers)
for server, server_info in dns_servers_copy.items():
status = dns_servers_status.get(server_info['ip_address'], None)
if (not status):
tls = 'Waiting'
dns = 'Waiting'
else:
dns = 'UP' if status['dns_up'] else 'Down'
tls = 'UP' if status['tls_up'] else 'Down'
if (not tls_enabled):
tls = 'Disabled'
dns_servers[server]['dns_up'] = dns
dns_servers[server]['tls_up'] = tls
# print(dnsstatus)
return dns_servers
def set_wan_interface(intf_type=INTF.DHCP):
'''
Change wan interface state between static or dhcp.
1. Configure interface type
2. Create netplan config from template
3. Move file to /etc/netplan
This does not configure an ip address of the interface when setting to static. see: set_wan_ip()
'''
# changing dhcp status of wan interface in config file.
with ConfigurationManager('config') as dnx:
interface_settings = dnx.load_configuration()
wan = interface_settings['interfaces']['builtins']['wan']
wan['state'] = intf_type
dnx.write_configuration(interface_settings)
# template used to generate yaml file with user configured fields
intf_template = load_configuration('intf_config',
filepath='dnx_system/interfaces')
# setting for static. removing dhcp4 and dhcp_overrides keys, then adding addresses with empty list
# NOTE: the ip configuration will unlock after the switch and can then be updated
if (intf_type is INTF.STATIC):
wan_intf = intf_template['network']['ethernets'][wan['ident']]
wan_intf.pop('dhcp4')
wan_intf.pop('dhcp4-overrides')
# initializing static, but not configuring an ip address
wan_intf['addresses'] = '[]'
# grabbing configured dns servers
dns_server_settings = load_configuration('dns_server')['resolvers']
dns1 = dns_server_settings['primary']['ip_address']
dns2 = dns_server_settings['secondary']['ip_address']
# dns server replacement in template required for static or dhcp
converted_config = json_to_yaml(intf_template)
converted_config = converted_config.replace('_PRIMARY__SECONDARY_',
f'{dns1},{dns2}')
# writing file into dnx_system folder due to limited permissions by the front end. netplan and the specific
# mv args are configured as sudo/no-pass to get the config to netplan and it applied without a restart.
with open(f'{HOME_DIR}/dnx_system/interfaces/01-dnx-interfaces.yaml',
'w') as dnx_intfs:
dnx_intfs.write(converted_config)
cmd_args = [
'{HOME_DIR}/dnx_system/interfaces/01-dnx-interfaces.yaml',
'/etc/netplan/01-dnx-interfaces.yaml'
]
system_action(module='webui', command='os.replace', args=cmd_args)
system_action(module='webui', command='netplan apply', args='')
def _load_restriction(self):
ip_proxy = load_configuration('ip_proxy')
logging = load_configuration('logging_client')
restriction_start = ip_proxy['time_restriction']['start']
restriction_length = ip_proxy['time_restriction']['length']
os_direction = logging['time_offset']['direction']
os_amount = logging['time_offset']['amount']
offset = int(f'{os_direction}{os_amount}') * ONE_DAY
return restriction_start, restriction_length, offset
def geolocation(region, rtype='country'):
region['cfg_dir'] = convert_int(region['cfg_dir'])
if (region['cfg_dir'] not in range(4)):
raise ValidationError(INVALID_FORM)
if (rtype == 'country'):
valid_regions = load_configuration('ip_proxy')['geolocation']
elif (rtype == 'continent'):
valid_regions = load_configuration('geolocation',
filepath='dnx_webui/data')
if region[rtype] not in valid_regions:
raise ValidationError(INVALID_FORM)
def set_wan_ip(wan_ip_settings):
'''
Modify configured WAN interface IP address.
1. Loads configured DNS servers
2. Loads wan interface identity
3. Create netplan config from template
4. Move file to /etc/netplan
'''
wan_int = load_configuration(
'config')['interfaces']['builtins']['wan']['ident']
# grabbing configured dns servers
dns_server_settings = load_configuration('dns_server')['resolvers']
dns1 = dns_server_settings['primary']['ip_address']
dns2 = dns_server_settings['secondary']['ip_address']
intf_template = load_configuration('intf_config',
filepath='dnx_system/interfaces')
# removing dhcp4 and dhcp_overrides keys, then adding ip address value
wan_intf = intf_template['network']['ethernets'][wan_int]
wan_intf.pop('dhcp4')
wan_intf.pop('dhcp4-overrides')
wan_intf[
'addresses'] = f'[{wan_ip_settings["ip"]}/{wan_ip_settings["cidr"]}]'
wan_intf['gateway4'] = f'{wan_ip_settings["dfg"]}'
converted_config = json_to_yaml(intf_template)
converted_config = converted_config.replace('_PRIMARY__SECONDARY_',
f'{dns1},{dns2}')
# writing file into dnx_system folder due to limited permissions by the front end. netplan and the specific
# mv args are configured as sudo/no-pass to get the config to netplan and it applied without a restart.
with open(f'{HOME_DIR}/dnx_system/interfaces/01-dnx-interfaces.yaml',
'w') as dnx_intfs:
dnx_intfs.write(converted_config)
cmd_args = [
f'{HOME_DIR}/dnx_system/interfaces/01-dnx-interfaces.yaml',
'/etc/netplan/01-dnx-interfaces.yaml'
]
system_action(module='webui', command='os.replace', args=cmd_args)
system_action(module='webui', command='netplan apply')
def get_settings(self, cfg_file):
syslog = load_configuration(cfg_file)['syslog']
SyslogService = self.SyslogService
SyslogService.syslog_enabled = syslog['enabled']
SyslogService.syslog_protocol = syslog['protocol']
SyslogService.tls_enabled = syslog['tls']['enabled']
SyslogService.self_signed_cert = syslog['tls']['self_signed']
SyslogService.tcp_fallback = syslog['tcp']['fallback']
SyslogService.udp_fallback = syslog['udp']['fallback']
syslog_servers = syslog['servers']
# if service is started without servers configured we will return here.
if not syslog_servers: return
names = ['primary', 'secondary']
with SyslogService.server_lock:
for name, cfg_server, mem_server in zip(names, syslog_servers.values(), SyslogService.syslog_servers):
if (cfg_server['ip_address'] == mem_server.get('ip')): continue
getattr(SyslogService.syslog_servers, name).update({
'ip': syslog_servers[name]['ip_address'],
PROTO.UDP: True, PROTO.TCP: True, PROTO.DNS_TLS: True
})
def _get_settings(self, cfg_file):
ip_proxy = load_configuration(cfg_file)
enabled = ip_proxy['time_restriction']['enabled']
self._change_attribute('_enabled', enabled)
self.initialize.done()
def _get_settings(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)
# updating user configuration items per interface in memory.
for settings in dhcp_settings['interfaces'].values():
# NOTE ex. ident: eth0, lo, enp0s3
intf_identity = settings['ident']
enabled = True if settings['enabled'] else False
# TODO: compare interface status in memory with what is loaded in. if it is different then the setting was just
# changed and needs to be acted on. implement register/unregister methods available to external callers and use
# them to act on the disable of an interfaces dhcp service. this should also be the most efficient in that if
# all listeners are disabled only the automate class will be actively processing on file changes.
# NOTE: .get is to cover server startup. do not change. test functionality.
sock_fd = self.DHCPServer.intf_settings[intf_identity]['fileno']
if (enabled and not self.DHCPServer.intf_settings[intf_identity].get('enabled', False)):
self.DHCPServer.enable(sock_fd, intf_identity)
elif (not enabled and self.DHCPServer.intf_settings[intf_identity].get('enabled', True)):
self.DHCPServer.disable(sock_fd, intf_identity)
# identity will be kept in settings just in case, though they key is the identity also.
self.DHCPServer.intf_settings[intf_identity].update(settings)
self.initialize.done()
def _get_reservations(self, cfg_file):
dhcp_settings = load_configuration(cfg_file)
# dict comp that retains all info of stored json data, but converts ip address into objects
self.DHCPServer.reservations = {
mac: {
'ip_address': IPv4Address(info['ip_address']),
'description': info['description']
}
for mac, info in dhcp_settings['reservations'].items()
}
# creating local reference for iteration performance
reservations = self.DHCPServer.reservations
# loaded all reserved ip addressing into a set to be referenced below
reserved_ips = set([IPv4Address(info['ip_address']) for info in reservations.values()])
# sets reserved ip addresses lease records to available is there are no longer configured
dhcp_leases = self.DHCPServer.leases
for ip, record in dhcp_leases.items():
# record[0] is record type. cross referencing ip reservation list with current lease table
# to reset any leased record placeholders for the reserved ip.
if (record[0] is DHCP.RESERVATION and IPv4Address(ip) not in reserved_ips):
dhcp_leases[ip] = _NULL_LEASE
# adding dhcp reservations to lease table to prevent them from being selected during an offer
self.DHCPServer.leases.update({
IPv4Address(info['ip_address']): (DHCP.RESERVATION, 0, mac) for mac, info in reservations.items()
})
self.initialize.done()
def load_interfaces(intf_type=INTF.BUILTINS, *, exclude=[]):
'''
return list of tuples of specified interface type.
[(intf_index, zone, ident)]
'''
intf_settings = load_configuration('config')['interfaces']
dnx_interfaces = intf_settings[intf_type.name.lower()]
# filtering out loopback during dict comprehension
system_interfaces = {v: k for k, v in if_nameindex()[1:]}
collected_intfs = []
if (intf_type is INTF.BUILTINS):
for intf_name, intf_info in dnx_interfaces.items():
ident = intf_info['ident']
zone = intf_info['zone']
intf_index = system_interfaces.get(ident)
if (not intf_index):
raise RuntimeError('failed to associate builtin <> system interfaces.')
if (intf_name not in exclude):
collected_intfs.append((intf_index, zone, ident))
else:
raise NotImplementedError('only builtin interfaces are currently supported.')
return collected_intfs
def wrapper(*args):
# will redirect to login page if user is not logged in
user = session.get('user', None)
if not user:
return redirect(url_for('dnx_login'))
# NOTE: this is dnx local tracking of sessions. not to be confused with flask session tracking. they
# are essentially copies of each other, but dnx is used to track all active sessions.
# NOTE: currently, dnx session data limits connections to 1 per user. this may change in the future, but
# some enterprise systems have similar restrictions or multiple tab restrictions.
session_tracker = load_configuration('session_tracker', filepath='dnx_webui/data')['active_users']
dnx_session_data = session_tracker.get(user['name'])
if (not dnx_session_data or dnx_session_data['remote_addr'] != request.remote_addr):
return redirect(url_for('dnx_login'))
# will redirect to not authorized page if the user role does not match
# requirements for the page
if (dnx_session_data['role'] not in authorized_roles):
session.pop('user', None)
return render_template('dnx_not_authorized.html', navi=True, login_btn=True, idle_timeout=False)
dnx_session_data[user['name']] = user
# flask page function
page_action = function_to_wrap(dnx_session_data)
return page_action
def load_page(form):
logging_settings = load_configuration('logging_client')
log = logging_settings['logging']
# correcting time for configured offset.
time_offset = logging_settings['time_offset']
system_time = System.format_date_time(fast_time())
local_time = System.calculate_time_offset(fast_time())
local_time = System.format_date_time(local_time)
logging_settings = {
'system': system_time,
'local': local_time,
'offset': {
'direction': time_offset['direction'],
'amount': time_offset['amount']
},
'logging': {
'log_levels': [level.title() for level in LOG_LEVELS],
'level': log['level'],
'length': log['length']
}
}
return logging_settings
def syslog_settings(syslog_settings):
syslog = load_configuration('syslog_client')
configured_syslog_servers = syslog['servers']
if (not configured_syslog_servers):
raise ValidationError(
'Syslog servers must be configured before modifying client settings.'
)
tls_retry = convert_int(syslog_settings['tls_retry'])
tcp_retry = convert_int(syslog_settings['tcp_retry'])
tls_settings = syslog_settings['tls']
syslog_settings = syslog_settings['syslog']
if (tls_retry not in [5, 10, 60] and tcp_retry not in [5, 10, 30]):
raise ValidationError('Syslog settings are not valid.')
for item in tls_settings:
if (item not in [
'enabled', 'tcp_fallback', 'udp_fallback', 'self_signed'
]):
raise ValidationError('Syslog settings are not valid.')
for item in syslog_settings:
if (item not in ['syslog_enabled', 'syslog_protocol']):
raise ValidationError('Syslog settings are not valid.')
if ('syslog_protocol' not in syslog_settings):
if ('encrypted_syslog' in tls_settings):
raise ValidationError('TCP must be enabled to enable TLS.')
if ('tcp_fallback' in tls_settings):
raise ValidationError('TLS must be enabled before TCP fallback.')
def combine_domains(Log):
dns_proxy = load_configuration('dns_proxy')
default_cats = dns_proxy['categories']['default']
ud_cats = dns_proxy['categories']['user_defined']
domain_signatures = []
# iterating over list of categories + DoH to load signature sets.
for cat in [*default_cats, 'dns-over-https']:
try:
with open(f'{HOME_DIR}/dnx_system/signatures/domain_lists/{cat}.domains', 'r') as file:
domain_signatures.extend([x.lower() for x in file.read().splitlines() if x and '#' not in x])
except FileNotFoundError:
Log.alert(f'signature file missing: {cat} domains.')
with open(f'{HOME_DIR}/dnx_system/signatures/domain_lists/blocked.domains', 'w+') as blocked:
blocked.write('\n'.join(domain_signatures))
# TODO: user defined categories will break the enum load on proxy / FIX
# looping over all user defined categories. ALSO. i think this will require a proxy restart if sigs change
for cat, settings in ud_cats:
if (not settings['enabled']): continue
# writing signatures to block file
for signature in settings[1:]:
blocked.write(f'{signature} {cat}\n'.lower())
# NOTE: nulling out signatures in memory so we dont have to wait for GC.
del domain_signatures
def load_page(section='MAIN'):
dnx_settings = load_configuration('config')
dnx_intfs = dnx_settings['interfaces']
dnx_zones = dnx_settings['zones']
# building out interface to zone map NOTE: builtins only for now
for intf_type in ['builtins', 'extended']:
for intf_name, intf_info in dnx_intfs[intf_type].items():
ident = intf_info['ident']
zone_map[intf_type][ident] = intf_name
# building zone list and reference counts NOTE: builtins only for now
for zone_type in ['builtins', 'user-defined']:
for zone_name, (zone_ident, zone_desc) in dnx_zones[zone_type].items():
# need to make converting zone ident/int to name easier in format function
_zone_map[zone_ident] = zone_name
zone_manager[zone_type][zone_name] = [
reference_counts[zone_ident], zone_desc
]
firewall_rules = get_and_format_rules(section)
return {
'zone_map': zone_map,
'zone_manager': zone_manager,
'firewall_rules': firewall_rules,
'pending_changes': is_pending_changes()
}
def monitor_rules(self, fw_rules):
'''calls to Cython are made from within this method block. the GIL must be manually acquired on the Cython
side or the Python interpreter will crash. Monitors the active firewall rules file for changes and loads
updates to cfirewall.'''
dnx_fw = load_configuration(fw_rules, filepath='dnx_system/iptables')
# splitting out sections then determine which one has changed. this is to reduce
# amount of work done on the C side. not for performance, but more for ease of programming.
# NOTE: index 1 start is needed because SYSTEM rules are held at index 0.
for i, section in enumerate(['BEFORE', 'MAIN', 'AFTER'], 1):
current_section = getattr(self, section)
new_section = dnx_fw[section]
# unchanged ruleset
if (current_section == new_section): continue
# updating ruleset to reflect changes
setattr(self, section, new_section)
# converting dict to list and each rule into a py array. this format is required due to
# transitioning between python and C. python arrays are compatible in C via memory views
# and Cython can handle the initial list.
ruleset = [array('L', rule) for rule in new_section.values()]
# NOTE: gil must be held throughout this call
error = self.cfirewall.update_ruleset(i, ruleset)
if (error):
pass # TODO: do something here
self._initialize.done()
def configure_interfaces():
interfaces_detected = check_system_interfaces()
user_intf_config = collect_interface_associations(interfaces_detected)
public_dns_servers = load_configuration('dns_server')['resolvers']
set_dnx_interfaces(user_intf_config)
set_dhcp_interfaces(user_intf_config)
with open(f'{SYSTEM_DIR}/interfaces/intf_config_template.json',
'r') as intf_configs:
intf_configs = intf_configs.read()
for intf_name, intf in user_intf_config.items():
intf_configs = intf_configs.replace(f'_{intf_name}_', intf)
# storing modified template containing specified interface names. this will be used to configure
# wan interface via webui or change system level dns servers.
write_configuration(json.loads(intf_configs),
'intf_config',
filepath='dnx_system/interfaces')
# setting public dns servers on the interface so the system itself will use the user configured
# servers in the web ui.
dns1 = public_dns_servers['primary']['ip_address']
dns2 = public_dns_servers['secondary']['ip_address']
yaml_output = json_to_yaml(intf_configs, is_string=True)
yaml_output = yaml_output.replace('_PRIMARY__SECONDARY_', f'{dns1},{dns2}')
write_net_config(yaml_output)
def dns_record_remove(dns_record_name):
dns_server = load_configuration('dns_server')
if (dns_record_name == 'dnx.firewall'):
raise ValidationError('Cannot remove dnxfirewall dns record.')
if (dns_record_name not in dns_server['records']):
raise ValidationError(INVALID_FORM)
def _load_leases(self):
dhcp_settings = load_configuration('dhcp_server')
stored_leases = dhcp_settings['leases']
self.update({
IPv4Address(ip): lease_info for ip, lease_info in stored_leases.items()
})
def get_settings(self, cfg_file):
# print('[+] Starting settings update poller.')
log_settings = load_configuration(cfg_file)
self.log_length = log_settings['logging']['length']
self.log_level = log_settings['logging']['level']
self._initialize.done()
def load_page(form):
users = load_configuration('logins', filepath='/dnx_webui/data')['users']
userlist = {}
for account, info in users.items():
userlist[account] = ('*****', info['role'])
return userlist
def load_infected_clients():
dhcp_server = load_configuration('dhcp_server')
users = dhcp_server['reservations']
return get_table_data(action='all',
table='infectedclients',
method='last',
users=users)
def _load_top_domains(self):
self._top_domains = load_configuration('dns_cache')
dom_list = reversed(list(self._top_domains))
self._dom_counter = Counter(
{dom: cnt
for cnt, dom in enumerate(dom_list)})
self._top_dom_filter = set(load_top_domains_filter())
def _user_authorized(self, username, hexpass):
local_accounts = load_configuration('logins', filepath='dnx_webui/data')['users']
try:
password = local_accounts[username]['password']
except KeyError:
return False
else:
# returning True on password match else False
return password == hexpass
def bandwidth():
intstat = {}
interface_bandwidth = load_configuration('interface_speed.json')
for interface, value in interface_bandwidth.items():
rx = str(round(int(value[0])/1024, 2)) + ' MB/s'
tx = str(round(int(value[1])/1024, 2)) + ' MB/s'
intstat[interface] = [rx, tx]
# print(intstat)
return intstat
def _log_settings(cls, cfg_file): # pylint: disable=no-self-argument
logging = load_configuration(cfg_file)
cls._LEVEL = logging['logging']['level']
cls._add_logging_methods()
# used to inform run method that it can return to caller. This is is only relevant on initial start.
cls._initialized = True
def domain_category_keywords(categories):
dns_proxy = load_configuration('dns_proxy')
domain_cats = dns_proxy['categories']['default']
for cat in categories:
if (cat not in domain_cats):
raise ValidationError(INVALID_FORM)
if (not domain_cats[cat]['enabled']):
raise ValidationError(INVALID_FORM)
def calculate_time_offset(logged_time):
'''returns modified time based on current time offset settings.'''
logging = load_configuration('logging_client')
offset = logging['time_offset']
os_direction = offset['direction']
os_amount = offset['amount']
offset = int(f'{os_direction}{os_amount}') * ONE_HOUR
return logged_time + offset
def load_page(form):
dnx_settings = load_configuration('config')
all_services = []
for service, desc in dnx_settings['services'].items():
status = True if Services.status(service) else False
service = ' '.join((service.split('-')[1:]))
all_services.append((service, desc, status))
return {'all_services': all_services, 'mgmt_access': dnx_settings['mgmt_access']}