import subprocess import threading import shutil import os from libnmap.parser import NmapParser, NmapParserException from sentry_sdk import capture_exception from natlas import screenshots from natlas import logging from natlas.net import NatlasNetworkServices from natlas.scanresult import ScanResult from natlas import utils logger = logging.get_logger("AgentThread") def command_builder(scan_id, agentConfig, target): outFiles = utils.get_data_dir(scan_id) + f"/nmap.{scan_id}" command = [ "nmap", "--privileged", "-oA", outFiles, "--servicedb", "./tmp/natlas-services" ] commandDict = { "versionDetection": "-sV", "osDetection": "-O", "osScanLimit": "--osscan-limit", "noPing": "-Pn", "onlyOpens": "--open", "udpScan": "-sUS",
from config import Config from natlas.threadscan import ThreadScan from natlas.net import NatlasNetworkServices ERR = { "INVALIDTARGET": 1, "SCANTIMEOUT": 2, "DATANOTFOUND": 3, "INVALIDDATA": 4 } config = Config() MAX_QUEUE_SIZE = (config.max_threads ) # only queue enough work for each of our active threads global_logger = logging.get_logger("MainThread") netsrv = NatlasNetworkServices(config) def add_targets_to_queue(target, q): targetNetwork = ipaddress.ip_network(target.strip()) if targetNetwork.num_addresses == 1: target_data = netsrv.get_work( target=str(targetNetwork.network_address)) if not target_data: return q.put(target_data) else: # Iterate over usable hosts in target, queue.put will block until a queue slot is available for t in targetNetwork.hosts(): target_data = netsrv.get_work(target=str(t))
class NatlasNetworkServices: config = None netlogger = logging.get_logger("NetworkServices") api_endpoints = { "GETSERVICES": "/api/natlas-services", "GETWORK": "/api/getwork", "SUBMIT": "/api/submit" } def __init__(self, config): self.config = config if self.config.ignore_ssl_warn: requests.packages.urllib3.disable_warnings(InsecureRequestWarning) def make_request(self, endpoint, reqType="GET", postData=None, contentType="application/json", statusCode=200): headers = { 'user-agent': 'natlas-agent/{}'.format(self.config.NATLAS_VERSION) } if self.config.agent_id and self.config.auth_token: authheader = self.config.agent_id + ":" + self.config.auth_token headers['Authorization'] = 'Bearer {}'.format(authheader) args = { "timeout": self.config.request_timeout, "headers": headers, "verify": not self.config.ignore_ssl_warn } try: if reqType == "GET": req = requests.get(self.config.server + endpoint, **args) if req.status_code == 200: if 'message' in req.json(): self.netlogger.info("[Server] " + req.json()['message']) return req if req.status_code == 403: if 'message' in req.json(): self.netlogger.error("[Server] " + req.json()['message']) if 'retry' in req.json() and not req.json()['retry']: os._exit(403) if req.status_code == 400: if 'message' in req.json(): self.netlogger.warn("[Server] " + req.json()['message']) return req if req.status_code != statusCode: self.netlogger.error("Expected %s, received %s" % (statusCode, req.status_code)) if 'message' in req.json(): self.netlogger.warn("[Server] " + req.json()['message']) return req if req.headers['content-type'] != contentType: self.netlogger.warn( "Expected %s, received %s" % (contentType, req.headers['content-type'])) return False elif reqType == "POST" and postData: args['json'] = postData req = requests.post(self.config.server + endpoint, **args) if req.status_code == 200: if 'message' in req.json(): self.netlogger.info("[Server] " + req.json()['message']) return req if req.status_code == 403: if 'message' in req.json(): self.netlogger.error("[Server] " + req.json()['message']) if 'retry' in req.json() and not req.json()['retry']: os._exit(403) if req.status_code == 400: if 'message' in req.json(): self.netlogger.warn("[Server] " + req.json()['message']) return req if req.status_code != statusCode: self.netlogger.warn("Expected %s, received %s" % (statusCode, req.status_code)) if 'message' in req.json(): self.netlogger.warn("[Server] " + req.json()['message']) return req except requests.ConnectionError: self.netlogger.warn("Connection Error connecting to %s" % self.config.server) return False except requests.Timeout: self.netlogger.warn("Request timed out after %s seconds." % self.config.request_timeout) return False except ValueError as e: self.netlogger.error("Error: %s" % e) return False return req def backoff_request(self, giveup=False, *args, **kwargs): attempt = 0 result = None while not result: result = self.make_request(*args, **kwargs) RETRY = False if result is False: # Retry if there are connection errors RETRY = True elif 'retry' in result.json() and result.json( )['retry']: # Retry if the server tells us to RETRY = True elif 'retry' in result.json() and not result.json( )['retry']: # Don't retry if the server tells us not to return result elif 'retry' not in result.json( ): # No instructions on whether to retry or not, so don't return result if RETRY: attempt += 1 if giveup and attempt == self.config.max_retries: self.netlogger.warn( "Request to %s failed %s times. Giving up" % (self.config.server, self.config.max_retries)) return False jitter = random.randint( 0, 1000) / 1000 # jitter to reduce chance of locking current_sleep = min( self.config.backoff_max, self.config.backoff_base * 2**attempt) + jitter self.netlogger.warn( "Request to %s failed. Waiting %s seconds before retrying." % (self.config.server, current_sleep)) time.sleep(current_sleep) return result def get_services_file(self): self.netlogger.info("Fetching natlas-services file from %s" % self.config.server) response = self.backoff_request( endpoint=self.api_endpoints["GETSERVICES"]) if response: serviceData = response.json() if serviceData["id"] == "None": self.netlogger.error("%s doesn't have a service file for us" % self.config.server) return False if not hashlib.sha256(serviceData["services"].encode()).hexdigest( ) == serviceData["sha256"]: self.netlogger.error( "hash provided by %s doesn't match locally computed hash of services" % self.config.server) return False with open("tmp/natlas-services", "w") as f: f.write(serviceData["services"]) with open("tmp/natlas-services", "r") as f: if not hashlib.sha256(f.read().rstrip( '\r\n').encode()).hexdigest() == serviceData["sha256"]: self.netlogger.error( "hash of local file doesn't match hash provided by server" ) return False else: return False # return false if we were unable to get a response from the server return serviceData[ "sha256"] # return True if we got a response and everything checks out def get_work(self, target=None): if target: self.netlogger.info("Getting work config for %s from %s" % (target, self.config.server)) get_work_endpoint = self.api_endpoints[ "GETWORK"] + "?target=" + target else: self.netlogger.info("Getting work from %s" % (self.config.server)) get_work_endpoint = self.api_endpoints["GETWORK"] response = self.backoff_request(endpoint=get_work_endpoint) if response: work = response.json() else: return False # failed to get work from server return work # Results is a ScanResult object def submit_results(self, results): if 'timed_out' in results.result and results.result['timed_out']: self.netlogger.info("Submitting scan timeout notice for %s" % results.result['ip']) elif 'is_up' in results.result and results.result[ 'is_up'] and 'port_count' in results.result and results.result[ 'port_count'] >= 0: self.netlogger.info( "Submitting %s ports for %s" % (results.result['port_count'], results.result['ip'])) elif not results.result['is_up']: self.netlogger.info("Submitting host down notice for %s" % (results.result['ip'])) else: self.netlogger.info("Submitting results for %s" % results.result["ip"]) response = self.backoff_request(giveup=True, endpoint=self.api_endpoints["SUBMIT"], reqType="POST", postData=json.dumps(results.result)) return response
#!/usr/bin/env python3 import subprocess import os import time from natlas import logging from natlas import utils logger = logging.get_logger("ScreenshotUtils") def get_web_screenshots(target, scan_id, services, proctimeout): data_dir = utils.get_data_dir(scan_id) outFiles = f"{data_dir}/aquatone.{scan_id}" inputstring = "" for service in services: inputstring += service + "://" + target + "\n" if inputstring: inputstring = inputstring[: -1] # trim trailing newline because otherwise chrome spits garbage into localhost for some reason logger.info("Attempting to take %s screenshot(s) for %s" % (', '.join(services).upper(), target)) p1 = subprocess.Popen(["echo", inputstring], stdout=subprocess.PIPE) # nosec process = subprocess.Popen( ["aquatone", "-scan-timeout", "2500", "-out", outFiles], stdin=p1.stdout,
import ipaddress import os import shutil from natlas import logging utillogger = logging.get_logger("Utilities") def validate_target(target, config): try: iptarget = ipaddress.ip_address(target) if iptarget.is_private and not config.scan_local: utillogger.error("We're not configured to scan local addresses!") return False except ipaddress.AddressValueError: utillogger.error("%s is not a valid IP Address" % target) return False return True def create_data_dir(scan_id): data_folder = f"data/natlas.{scan_id}" os.makedirs(data_folder, exist_ok=True) def get_data_dir(scan_id): data_folder = f"data/natlas.{scan_id}" return data_folder