def _register_new_socket(self): #, client_query=None):
with self._DNSServer.server_lock:
for tls_server in self._DNSServer.dns_servers:
# skipping over known down server
if (not tls_server[self._protocol]): continue
# attempting to connect via tls. if successful will return True, otherwise mark server as
# down and try next server.
if self._tls_connect(tls_server['ip']): return True
self.mark_server_down()
else:
self._DNSServer.tls_up = False
# NOTE: i dont think this gets hit anymore??? investigate after two server fails,
# the client would have already asked again. this would probably not help anything
# even if it technically did get hit sometimes. after tls is marked down they will
# be pushed over to fallback by queue.
# sending to fallback relay(udp) if enabled and client_query is present
# if (self._DNSServer.udp_fallback and client_query):
# self._send_to_fallback(client_query)
Log.error('NO SECURE SERVERS AVAILABLE!')
def _recv_handler(self):
recv_buffer = []
while True:
try:
data_from_server = self._relay_conn.sock.recv(1024)
except (socket.timeout, OSError) as e:
Log.dprint(f'RECV HANDLER: {e}')
break
else:
self._reset_fail_detection()
if (not data_from_server):
Log.dprint(
'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!')
break
recv_buffer.append(data_from_server)
while recv_buffer:
current_data = b''.join(recv_buffer)[2:]
data_len = short_unpackf(recv_buffer[0])[0]
if (len(current_data) == data_len):
recv_buffer = []
elif (len(current_data) > data_len):
recv_buffer = [current_data[data_len:]]
else:
break
if not self.is_keepalive(current_data):
self.DNSServer.responder.add(current_data[:data_len])
self._relay_conn.sock.close()
def _register_new_socket(self):
with self.DNSServer.server_lock:
for dns_server in self.DNSServer.dns_servers:
if (not dns_server[self._protocol]): continue
return self._create_socket(
dns_server['ip']) # never fail so will always return True
else:
Log.critical('NO UDP SERVER AVAILABLE.')
def _setup(cls):
dns_sigs = Configuration.load_signatures()
# en_dns | dns | tld | keyword
cls.signatures = SIGNATURES(set(), dns_sigs, {}, [])
Configuration.proxy_setup(cls)
cls.set_proxy_callback(func=Inspect.dns)
Log.notice(f'{cls.__name__} initialization complete.')
def _reset_flag(self, cache_type):
setattr(self, f'clear_{cache_type}', False)
with fo.ConfigurationManager('dns_server') as dnx:
dns_settings = dnx.load_configuration()
dns_settings['dns_server']['cache'][cache_type] = False
dnx.write_configuration(dns_settings)
Log.notice(f'{cache_type.replace("_", " ")} has been cleared.')
def _register_new_socket(self, client_query=None):
with self.DNSServer.server_lock:
for tls_server in self.DNSServer.dns_servers:
if (not tls_server[self._protocol]): continue
if self._tls_connect(tls_server['ip']): return True
self.mark_server_down()
else:
Log.error('NO SECURE SERVERS AVAILABLE!')
self.DNSServer.tls_up = False
if (self.DNSServer.udp_fallback and client_query):
self._send_to_fallback(client_query)
def dns(cls, packet):
self = cls(packet)
request_results = self._dns_inspect()
# NOTE: accessing class var through instance is 7-10% faster
if (not request_results.redirect):
self._Proxy.notify_server(packet, decision=DNS.ALLOWED)
else:
self._Proxy.notify_server(packet, decision=DNS.FLAGGED)
packet.generate_proxy_response()
self._Proxy.send_to_client(packet)
Log.log(packet, request_results)
def _tls_connect(self, tls_server):
Log.dprint(f'Opening Secure socket to {tls_server}: 853')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
dns_sock = self._tls_context.wrap_socket(sock,
server_hostname=tls_server)
try:
dns_sock.connect((tls_server, PROTO.DNS_TLS))
except OSError:
return None
else:
self._relay_conn = RELAY_CONN(tls_server, dns_sock, dns_sock.send,
dns_sock.recv, dns_sock.version())
return True
def _request_queue(self):
return_ready = self.REQ_TRACKER.return_ready
while True:
# this blocks until request tracker returns (at least 1 client query has been inspected)
requests = return_ready()
for client_query, decision in requests:
if decision is DNS.ALLOWED and not self._cached_response(
client_query):
self._handle_query(client_query)
Log.informational(
f'{self.protocol.name} Relay ALLOWED | {client_query}') # pylint: disable=no-member
def _recv_handler(self, recv_buffer=[]):
recv_buff_append = recv_buffer.append
recv_buff_clear = recv_buffer.clear
conn_recv = self._relay_conn.recv
responder_add = self._DNSServer.responder.add
while True:
try:
data_from_server = conn_recv(2048)
# TODO: i feel like this has to do a lookup everytime. if that is the case we should directly reference timeout
except (socket.timeout, OSError) as e:
Log.dprint(f'RECV HANDLER: {e}')
break
else:
self._reset_fail_detection()
# if no data is received/EOF the remote end has closed the connection
if (not data_from_server):
Log.dprint(
'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!')
break
recv_buff_append(data_from_server)
while recv_buffer:
current_data = byte_join(recv_buffer)
data_len, data = short_unpackf(
current_data)[0], current_data[2:]
# more data is needed for a complete response. NOTE: this scenario is kind of dumb
# and shouldnt happen unless the server sends length of record and record seperately.
if (len(data) < data_len): break
# clearing the buffer since we either have nothing left to process or we will re add
# the leftover bytes back with the next condition.
recv_buff_clear()
# if expected data length is greater than local buffer, multiple records were returned
# in a batch so appending leftover bytes after removing the current records data from buffer.
if (len(data) > data_len):
recv_buff_append(data[data_len:])
# ignoring internally generated connection keepalives
if (data[0] != DNS.KEEPALIVE):
responder_add(data[:data_len])
self._relay_conn.sock.close()
def _recv_handler(self, recv_buffer=[]):
recv_buff_append = recv_buffer.append
recv_buff_clear = recv_buffer.clear
conn_recv = self._relay_conn.recv
responder_add = self._DNSServer.responder.add
while True:
try:
data_from_server = conn_recv(2048)
# TODO: i feel like this has to do a lookup everytime. if that is the case we should directly reference timeout
except (socket.timeout, OSError) as e:
Log.dprint(f'RECV HANDLER: {e}')
break
else:
self._reset_fail_detection()
# if no data is received/EOF the remote end has closed the connection
if (not data_from_server):
Log.dprint(
'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!')
break
recv_buff_append(data_from_server)
while recv_buffer:
current_data = byte_join(recv_buffer)
data_len, data = short_unpackf(
current_data)[0], current_data[2:]
# more data is needed for a complete response.
if data_len < len(data): break
# clearing the buffer. this is the easiest way to deal with unkown condition of single or multiple
# dns records contained in one packet.
recv_buff_clear()
# if identified data length is > the actual data we have, multiple records are contained in the packet
# so we will append the remainder back into the buffer.
if data_len > len(data):
recv_buff_append(data[data_len:])
# filtering internal connection keepalives
if (data[0] != DNS.KEEPALIVE):
responder_add(data[:data_len])
self._relay_conn.sock.close()
def _tls_connect(self, tls_server):
Log.dprint(f'Opening Secure socket to {tls_server}: 853')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
dns_sock = self._tls_context.wrap_socket(sock,
server_hostname=tls_server)
try:
dns_sock.connect((tls_server, PROTO.DNS_TLS))
except OSError:
return None
else:
return True
# NOTE: is this ok if we fail to connect? seems alittle weird after looking at it again.
finally:
self._relay_conn = RELAY_CONN(tls_server, dns_sock, dns_sock.send,
dns_sock.recv, dns_sock.version())
def _tls_connect(self, tls_server):
Log.dprint(f'Opening Secure socket to {tls_server}: 853')
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# NOTE: this should improve sending performance since we expect a dns record to only be a small
# portion of available bytes in MTU/max bytes(1500). seems to provide no improvement after 1 run.
# there could be other bottlenecks in play so we can re evaluate later.
# sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
dns_sock = self._tls_context.wrap_socket(sock,
server_hostname=tls_server)
try:
dns_sock.connect((tls_server, PROTO.DNS_TLS))
except OSError:
return None
else:
return True
finally:
self._relay_conn = RELAY_CONN(tls_server, dns_sock)
def _wait_for_proxy_decision(self, client_query):
# waiting for proxy decision. if iteration completes normally, it will be marked as a timeout.
# NOTE: TESTING | after each check a msec will get added to the interval.
for interval in [x / 1000
for x in range(DNS.WAIT_COUNT)]: # converting to msec
decision = self._req_results_pop(client_query.address,
DNS.NO_NOTICE)
if (decision is DNS.FLAGGED): return
if (decision is DNS.ALLOWED): break
sleep(interval)
else:
return
Log.informational(
f'{self.protocol.name} Relay ALLOWED | {client_query}') # pylint: disable=no-member
if not self._cached_response(client_query):
self._handle_query(client_query)
def udp(self):
if (not self.is_enabled and not self.DNSServer.udp_fallback):
return TEN_SEC
with self.DNSServer.server_lock:
for server in self.DNSServer.dns_servers:
if (server[self._protocol]):
continue # not checking if server/proto is known up
if self._udp_reachable(server['ip']):
server[PROTO.UDP] = True
Log.notice('DNS server {} has recovered on {}.'.format(
server['ip'], self._protocol.name))
write_configuration(self.DNSServer.dns_servers._asdict(),
'dns_server_status')
return THIRTY_SEC
def tls(self):
if (not self.is_enabled): return TEN_SEC
with self.DNSServer.server_lock:
for secure_server in self.DNSServer.dns_servers:
if (secure_server[self._protocol]):
continue # not checking if server/proto is known up
if self._tls_reachable(secure_server):
secure_server[PROTO.DNS_TLS] = True,
self.DNSServer.tls_up = True
Log.notice('DNS server {} has recovered on {}.'.format(
secure_server['ip'], self._protocol.name))
if (self.DNSServer.tls_up):
write_configuration(self.DNSServer.dns_servers._asdict(),
'dns_server_status')
return THIRTY_SEC
def udp(self):
if (not self.is_enabled and not self.DNSServer.udp_fallback):
return TEN_SEC
DNSServer = self.DNSServer
with DNSServer.server_lock:
for server in DNSServer.dns_servers:
# no check needed if server/proto is known up
if (server[self._protocol]): continue
# if server responds to connection attempt, it will be marked as available
if self._udp_reachable(server['ip']):
server[PROTO.UDP] = True
Log.notice('DNS server {} has recovered on {}.'.format(
server['ip'], self._protocol.name))
write_configuration(self.DNSServer.dns_servers._asdict(),
'dns_server_status')
return THIRTY_SEC
def _dns_inspect(self):
packet, Proxy, whitelisted = self._packet, self._Proxy, False
# TODO: make this only apply to global whitelist as currently it will think tor whitelist entries
# are part of it.
# checking whitelist.
if (packet.src_ip in Proxy.whitelist.ip):
whitelisted = True
# NOTE: dns whitelist does not override tld blocks at the moment
# signature/ blacklist check. if either match will return results
for i, enum_request in enumerate(packet.requests):
# TLD (top level domain) block | after first index will pass
# nested to allow for continue
if (not i):
if Proxy.signatures.tld.get(enum_request):
Log.dprint(f'TLD Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'tld filter',
enum_request)
continue
# NOTE: allowing malicious category overrides (for false positives)
if (enum_request in Proxy.whitelist.dns):
return DNS_REQUEST_RESULTS(False, None, None)
# ip whitelist overrides configured blacklist
if (not whitelisted and enum_request in Proxy.blacklist.dns):
Log.dprint(f'Blacklist Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'blacklist', 'time based')
# pulling domain category if signature present.
category = self._bin_search(enum_request)
if category and self._block_query(category, whitelisted):
Log.dprint(f'Category Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'category', category)
# Keyword search within domain || block if match
for keyword, category in Proxy.signatures.keyword:
if (keyword in packet.request):
Log.dprint(f'Keyword Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'keyword', category)
# DEFAULT ACTION | ALLOW
return DNS_REQUEST_RESULTS(False, None, None)
def tls(self):
if (not self.is_enabled): return TEN_SEC
DNSServer = self.DNSServer
with DNSServer.server_lock:
for secure_server in DNSServer.dns_servers:
# no check needed if server/proto is known up
if (secure_server[self._protocol]): continue
# if server responds to connection attempt, it will be marked as available
if self._tls_reachable(secure_server):
secure_server[PROTO.DNS_TLS] = True,
DNSServer.tls_up = True
Log.notice('DNS server {} has recovered on {}.'.format(
secure_server['ip'], self._protocol.name))
# will write server status change individually as its unlikely both will be down at same time
write_configuration(DNSServer.dns_servers._asdict(),
'dns_server_status')
return THIRTY_SEC
def _dns_inspect(self, packet):
Proxy = self._Proxy # NOTE: consider sending this in on all inspection classes?
whitelisted = self._ip_whitelist_get(packet.src_ip, False)
# signature/ blacklist check.
# DNS_REQUEST_RESULTS(redirect, block type, category)
# NOTE: dns whitelist does not override tld blocks at the moment | this is most likely the desired setup
for i, enum_request in enumerate(packet.requests):
# TLD (top level domain) block | after first index will pass nested to allow for continue
if (not i):
if self._tld_get(enum_request):
Log.dprint(f'TLD Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'tld filter',
enum_request)
continue
# NOTE: allowing malicious category overrides (for false positives)
if (enum_request in Proxy.whitelist.dns):
return DNS_REQUEST_RESULTS(False, None, None)
# ip whitelist overrides configured blacklist
if (not whitelisted and enum_request in Proxy.blacklist.dns):
Log.dprint(f'Blacklist Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'blacklist', 'time based')
# pulling domain category if signature present. | NOTE: this is now using imported cython function factory
category = DNS_CAT(_recursive_binary_search(enum_request))
if (category is not DNS_CAT.NONE) and self._block_query(
category, whitelisted):
Log.dprint(f'Category Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'category', category)
# Keyword search within domain || block if match
for keyword, category in Proxy.signatures.keyword:
if (keyword in packet.request):
Log.dprint(f'Keyword Block: {packet.request}')
return DNS_REQUEST_RESULTS(True, 'keyword', category)
# DEFAULT ACTION | ALLOW
return DNS_REQUEST_RESULTS(False, None, None)
return DNS_REQUEST_RESULTS(False, None, None)
# # grabbing the request category and determining whether the request should be blocked. if so, returns general
# # information for further processing
def _block_query(self, category, whitelisted):
# signature match, but blocking disabled for the category | ALLOW
if (category not in self._Proxy.signatures.en_dns):
return False
# signature match, not whitelisted, or whitelisted and cat is bad | BLOCK
if (not whitelisted
or category in [DNS_CAT.malicious, DNS_CAT.cryptominer]):
return True
# default action | ALLOW
return False
if __name__ == '__main__':
dns_cat_signatures = Configuration.load_dns_signature_bitmap()
# using cython function factory to create binary search function with module specific signatures
signature_bounds = (0, len(dns_cat_signatures) - 1)
_recursive_binary_search = generate_recursive_binary_search(
dns_cat_signatures, signature_bounds)
Log.run(name=LOG_NAME)
DNSProxy.run(Log, threaded=True)
DNSServer.run(Log, threaded=False)
right = mid - 1
else:
return None
self._match = match
# on bin match, recursively call to check host ids
if (not recursion):
return self._bin_search((rh_id, 0), recursion=True)
return DNS_CAT(match)
# grabbing the request category and determining whether the request should be blocked. if so, returns general
# information for further processing
def _block_query(self, category, whitelisted):
# signature match, but blocking disabled for the category | ALLOW
if (category not in self._Proxy.signatures.en_dns):
return False
# signature match, not whitelisted, or whitelisted and cat is bad | BLOCK
if (not whitelisted or category in ['malicious', 'cryptominer']):
return True
# default action | ALLOW
return False
if __name__ == '__main__':
Log.run(name=LOG_NAME, verbose=VERBOSE, root=ROOT)
DNSProxy.run(Log, threaded=True)
DNSServer.run(Log, threaded=True)
def add(self, request, data_to_cache):
'''add query to cache after calculating expiration time.'''
self[request] = data_to_cache
Log.dprint(f'CACHE ADD | NAME: {request} TTL: {data_to_cache.ttl}')
def _setup(cls):
Configuration.proxy_setup(cls)
cls.set_proxy_callback(func=Inspect.dns)
Log.notice(f'{cls.__name__} initialization complete.')