def _register_new_socket(self): #, client_query=None): with self._DNSServer.server_lock: for tls_server in self._DNSServer.dns_servers: # skipping over known down server if (not tls_server[self._protocol]): continue # attempting to connect via tls. if successful will return True, otherwise mark server as # down and try next server. if self._tls_connect(tls_server['ip']): return True self.mark_server_down() else: self._DNSServer.tls_up = False # NOTE: i dont think this gets hit anymore??? investigate after two server fails, # the client would have already asked again. this would probably not help anything # even if it technically did get hit sometimes. after tls is marked down they will # be pushed over to fallback by queue. # sending to fallback relay(udp) if enabled and client_query is present # if (self._DNSServer.udp_fallback and client_query): # self._send_to_fallback(client_query) Log.error('NO SECURE SERVERS AVAILABLE!')
def _recv_handler(self): recv_buffer = [] while True: try: data_from_server = self._relay_conn.sock.recv(1024) except (socket.timeout, OSError) as e: Log.dprint(f'RECV HANDLER: {e}') break else: self._reset_fail_detection() if (not data_from_server): Log.dprint( 'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!') break recv_buffer.append(data_from_server) while recv_buffer: current_data = b''.join(recv_buffer)[2:] data_len = short_unpackf(recv_buffer[0])[0] if (len(current_data) == data_len): recv_buffer = [] elif (len(current_data) > data_len): recv_buffer = [current_data[data_len:]] else: break if not self.is_keepalive(current_data): self.DNSServer.responder.add(current_data[:data_len]) self._relay_conn.sock.close()
def _register_new_socket(self): with self.DNSServer.server_lock: for dns_server in self.DNSServer.dns_servers: if (not dns_server[self._protocol]): continue return self._create_socket( dns_server['ip']) # never fail so will always return True else: Log.critical('NO UDP SERVER AVAILABLE.')
def _setup(cls): dns_sigs = Configuration.load_signatures() # en_dns | dns | tld | keyword cls.signatures = SIGNATURES(set(), dns_sigs, {}, []) Configuration.proxy_setup(cls) cls.set_proxy_callback(func=Inspect.dns) Log.notice(f'{cls.__name__} initialization complete.')
def _reset_flag(self, cache_type): setattr(self, f'clear_{cache_type}', False) with fo.ConfigurationManager('dns_server') as dnx: dns_settings = dnx.load_configuration() dns_settings['dns_server']['cache'][cache_type] = False dnx.write_configuration(dns_settings) Log.notice(f'{cache_type.replace("_", " ")} has been cleared.')
def _register_new_socket(self, client_query=None): with self.DNSServer.server_lock: for tls_server in self.DNSServer.dns_servers: if (not tls_server[self._protocol]): continue if self._tls_connect(tls_server['ip']): return True self.mark_server_down() else: Log.error('NO SECURE SERVERS AVAILABLE!') self.DNSServer.tls_up = False if (self.DNSServer.udp_fallback and client_query): self._send_to_fallback(client_query)
def dns(cls, packet): self = cls(packet) request_results = self._dns_inspect() # NOTE: accessing class var through instance is 7-10% faster if (not request_results.redirect): self._Proxy.notify_server(packet, decision=DNS.ALLOWED) else: self._Proxy.notify_server(packet, decision=DNS.FLAGGED) packet.generate_proxy_response() self._Proxy.send_to_client(packet) Log.log(packet, request_results)
def _tls_connect(self, tls_server): Log.dprint(f'Opening Secure socket to {tls_server}: 853') sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dns_sock = self._tls_context.wrap_socket(sock, server_hostname=tls_server) try: dns_sock.connect((tls_server, PROTO.DNS_TLS)) except OSError: return None else: self._relay_conn = RELAY_CONN(tls_server, dns_sock, dns_sock.send, dns_sock.recv, dns_sock.version()) return True
def _request_queue(self): return_ready = self.REQ_TRACKER.return_ready while True: # this blocks until request tracker returns (at least 1 client query has been inspected) requests = return_ready() for client_query, decision in requests: if decision is DNS.ALLOWED and not self._cached_response( client_query): self._handle_query(client_query) Log.informational( f'{self.protocol.name} Relay ALLOWED | {client_query}') # pylint: disable=no-member
def _recv_handler(self, recv_buffer=[]): recv_buff_append = recv_buffer.append recv_buff_clear = recv_buffer.clear conn_recv = self._relay_conn.recv responder_add = self._DNSServer.responder.add while True: try: data_from_server = conn_recv(2048) # TODO: i feel like this has to do a lookup everytime. if that is the case we should directly reference timeout except (socket.timeout, OSError) as e: Log.dprint(f'RECV HANDLER: {e}') break else: self._reset_fail_detection() # if no data is received/EOF the remote end has closed the connection if (not data_from_server): Log.dprint( 'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!') break recv_buff_append(data_from_server) while recv_buffer: current_data = byte_join(recv_buffer) data_len, data = short_unpackf( current_data)[0], current_data[2:] # more data is needed for a complete response. NOTE: this scenario is kind of dumb # and shouldnt happen unless the server sends length of record and record seperately. if (len(data) < data_len): break # clearing the buffer since we either have nothing left to process or we will re add # the leftover bytes back with the next condition. recv_buff_clear() # if expected data length is greater than local buffer, multiple records were returned # in a batch so appending leftover bytes after removing the current records data from buffer. if (len(data) > data_len): recv_buff_append(data[data_len:]) # ignoring internally generated connection keepalives if (data[0] != DNS.KEEPALIVE): responder_add(data[:data_len]) self._relay_conn.sock.close()
def _recv_handler(self, recv_buffer=[]): recv_buff_append = recv_buffer.append recv_buff_clear = recv_buffer.clear conn_recv = self._relay_conn.recv responder_add = self._DNSServer.responder.add while True: try: data_from_server = conn_recv(2048) # TODO: i feel like this has to do a lookup everytime. if that is the case we should directly reference timeout except (socket.timeout, OSError) as e: Log.dprint(f'RECV HANDLER: {e}') break else: self._reset_fail_detection() # if no data is received/EOF the remote end has closed the connection if (not data_from_server): Log.dprint( 'RECV HANDLER: PIPELINE CLOSED BY REMOTE SERVER!') break recv_buff_append(data_from_server) while recv_buffer: current_data = byte_join(recv_buffer) data_len, data = short_unpackf( current_data)[0], current_data[2:] # more data is needed for a complete response. if data_len < len(data): break # clearing the buffer. this is the easiest way to deal with unkown condition of single or multiple # dns records contained in one packet. recv_buff_clear() # if identified data length is > the actual data we have, multiple records are contained in the packet # so we will append the remainder back into the buffer. if data_len > len(data): recv_buff_append(data[data_len:]) # filtering internal connection keepalives if (data[0] != DNS.KEEPALIVE): responder_add(data[:data_len]) self._relay_conn.sock.close()
def _tls_connect(self, tls_server): Log.dprint(f'Opening Secure socket to {tls_server}: 853') sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dns_sock = self._tls_context.wrap_socket(sock, server_hostname=tls_server) try: dns_sock.connect((tls_server, PROTO.DNS_TLS)) except OSError: return None else: return True # NOTE: is this ok if we fail to connect? seems alittle weird after looking at it again. finally: self._relay_conn = RELAY_CONN(tls_server, dns_sock, dns_sock.send, dns_sock.recv, dns_sock.version())
def _tls_connect(self, tls_server): Log.dprint(f'Opening Secure socket to {tls_server}: 853') sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # NOTE: this should improve sending performance since we expect a dns record to only be a small # portion of available bytes in MTU/max bytes(1500). seems to provide no improvement after 1 run. # there could be other bottlenecks in play so we can re evaluate later. # sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) dns_sock = self._tls_context.wrap_socket(sock, server_hostname=tls_server) try: dns_sock.connect((tls_server, PROTO.DNS_TLS)) except OSError: return None else: return True finally: self._relay_conn = RELAY_CONN(tls_server, dns_sock)
def _wait_for_proxy_decision(self, client_query): # waiting for proxy decision. if iteration completes normally, it will be marked as a timeout. # NOTE: TESTING | after each check a msec will get added to the interval. for interval in [x / 1000 for x in range(DNS.WAIT_COUNT)]: # converting to msec decision = self._req_results_pop(client_query.address, DNS.NO_NOTICE) if (decision is DNS.FLAGGED): return if (decision is DNS.ALLOWED): break sleep(interval) else: return Log.informational( f'{self.protocol.name} Relay ALLOWED | {client_query}') # pylint: disable=no-member if not self._cached_response(client_query): self._handle_query(client_query)
def udp(self): if (not self.is_enabled and not self.DNSServer.udp_fallback): return TEN_SEC with self.DNSServer.server_lock: for server in self.DNSServer.dns_servers: if (server[self._protocol]): continue # not checking if server/proto is known up if self._udp_reachable(server['ip']): server[PROTO.UDP] = True Log.notice('DNS server {} has recovered on {}.'.format( server['ip'], self._protocol.name)) write_configuration(self.DNSServer.dns_servers._asdict(), 'dns_server_status') return THIRTY_SEC
def tls(self): if (not self.is_enabled): return TEN_SEC with self.DNSServer.server_lock: for secure_server in self.DNSServer.dns_servers: if (secure_server[self._protocol]): continue # not checking if server/proto is known up if self._tls_reachable(secure_server): secure_server[PROTO.DNS_TLS] = True, self.DNSServer.tls_up = True Log.notice('DNS server {} has recovered on {}.'.format( secure_server['ip'], self._protocol.name)) if (self.DNSServer.tls_up): write_configuration(self.DNSServer.dns_servers._asdict(), 'dns_server_status') return THIRTY_SEC
def udp(self): if (not self.is_enabled and not self.DNSServer.udp_fallback): return TEN_SEC DNSServer = self.DNSServer with DNSServer.server_lock: for server in DNSServer.dns_servers: # no check needed if server/proto is known up if (server[self._protocol]): continue # if server responds to connection attempt, it will be marked as available if self._udp_reachable(server['ip']): server[PROTO.UDP] = True Log.notice('DNS server {} has recovered on {}.'.format( server['ip'], self._protocol.name)) write_configuration(self.DNSServer.dns_servers._asdict(), 'dns_server_status') return THIRTY_SEC
def _dns_inspect(self): packet, Proxy, whitelisted = self._packet, self._Proxy, False # TODO: make this only apply to global whitelist as currently it will think tor whitelist entries # are part of it. # checking whitelist. if (packet.src_ip in Proxy.whitelist.ip): whitelisted = True # NOTE: dns whitelist does not override tld blocks at the moment # signature/ blacklist check. if either match will return results for i, enum_request in enumerate(packet.requests): # TLD (top level domain) block | after first index will pass # nested to allow for continue if (not i): if Proxy.signatures.tld.get(enum_request): Log.dprint(f'TLD Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'tld filter', enum_request) continue # NOTE: allowing malicious category overrides (for false positives) if (enum_request in Proxy.whitelist.dns): return DNS_REQUEST_RESULTS(False, None, None) # ip whitelist overrides configured blacklist if (not whitelisted and enum_request in Proxy.blacklist.dns): Log.dprint(f'Blacklist Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'blacklist', 'time based') # pulling domain category if signature present. category = self._bin_search(enum_request) if category and self._block_query(category, whitelisted): Log.dprint(f'Category Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'category', category) # Keyword search within domain || block if match for keyword, category in Proxy.signatures.keyword: if (keyword in packet.request): Log.dprint(f'Keyword Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'keyword', category) # DEFAULT ACTION | ALLOW return DNS_REQUEST_RESULTS(False, None, None)
def tls(self): if (not self.is_enabled): return TEN_SEC DNSServer = self.DNSServer with DNSServer.server_lock: for secure_server in DNSServer.dns_servers: # no check needed if server/proto is known up if (secure_server[self._protocol]): continue # if server responds to connection attempt, it will be marked as available if self._tls_reachable(secure_server): secure_server[PROTO.DNS_TLS] = True, DNSServer.tls_up = True Log.notice('DNS server {} has recovered on {}.'.format( secure_server['ip'], self._protocol.name)) # will write server status change individually as its unlikely both will be down at same time write_configuration(DNSServer.dns_servers._asdict(), 'dns_server_status') return THIRTY_SEC
def _dns_inspect(self, packet): Proxy = self._Proxy # NOTE: consider sending this in on all inspection classes? whitelisted = self._ip_whitelist_get(packet.src_ip, False) # signature/ blacklist check. # DNS_REQUEST_RESULTS(redirect, block type, category) # NOTE: dns whitelist does not override tld blocks at the moment | this is most likely the desired setup for i, enum_request in enumerate(packet.requests): # TLD (top level domain) block | after first index will pass nested to allow for continue if (not i): if self._tld_get(enum_request): Log.dprint(f'TLD Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'tld filter', enum_request) continue # NOTE: allowing malicious category overrides (for false positives) if (enum_request in Proxy.whitelist.dns): return DNS_REQUEST_RESULTS(False, None, None) # ip whitelist overrides configured blacklist if (not whitelisted and enum_request in Proxy.blacklist.dns): Log.dprint(f'Blacklist Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'blacklist', 'time based') # pulling domain category if signature present. | NOTE: this is now using imported cython function factory category = DNS_CAT(_recursive_binary_search(enum_request)) if (category is not DNS_CAT.NONE) and self._block_query( category, whitelisted): Log.dprint(f'Category Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'category', category) # Keyword search within domain || block if match for keyword, category in Proxy.signatures.keyword: if (keyword in packet.request): Log.dprint(f'Keyword Block: {packet.request}') return DNS_REQUEST_RESULTS(True, 'keyword', category) # DEFAULT ACTION | ALLOW return DNS_REQUEST_RESULTS(False, None, None)
return DNS_REQUEST_RESULTS(False, None, None) # # grabbing the request category and determining whether the request should be blocked. if so, returns general # # information for further processing def _block_query(self, category, whitelisted): # signature match, but blocking disabled for the category | ALLOW if (category not in self._Proxy.signatures.en_dns): return False # signature match, not whitelisted, or whitelisted and cat is bad | BLOCK if (not whitelisted or category in [DNS_CAT.malicious, DNS_CAT.cryptominer]): return True # default action | ALLOW return False if __name__ == '__main__': dns_cat_signatures = Configuration.load_dns_signature_bitmap() # using cython function factory to create binary search function with module specific signatures signature_bounds = (0, len(dns_cat_signatures) - 1) _recursive_binary_search = generate_recursive_binary_search( dns_cat_signatures, signature_bounds) Log.run(name=LOG_NAME) DNSProxy.run(Log, threaded=True) DNSServer.run(Log, threaded=False)
right = mid - 1 else: return None self._match = match # on bin match, recursively call to check host ids if (not recursion): return self._bin_search((rh_id, 0), recursion=True) return DNS_CAT(match) # grabbing the request category and determining whether the request should be blocked. if so, returns general # information for further processing def _block_query(self, category, whitelisted): # signature match, but blocking disabled for the category | ALLOW if (category not in self._Proxy.signatures.en_dns): return False # signature match, not whitelisted, or whitelisted and cat is bad | BLOCK if (not whitelisted or category in ['malicious', 'cryptominer']): return True # default action | ALLOW return False if __name__ == '__main__': Log.run(name=LOG_NAME, verbose=VERBOSE, root=ROOT) DNSProxy.run(Log, threaded=True) DNSServer.run(Log, threaded=True)
def add(self, request, data_to_cache): '''add query to cache after calculating expiration time.''' self[request] = data_to_cache Log.dprint(f'CACHE ADD | NAME: {request} TTL: {data_to_cache.ttl}')
def _setup(cls): Configuration.proxy_setup(cls) cls.set_proxy_callback(func=Inspect.dns) Log.notice(f'{cls.__name__} initialization complete.')