def __init__(self, queue, config, auto=False, servicesSha=''): threading.Thread.__init__(self) self.queue = queue self.auto = auto self.servicesSha = servicesSha self.config = config self.netsrv = NatlasNetworkServices(self.config)
class ThreadScan(threading.Thread): def __init__(self, queue, config, auto=False, servicesSha=''): threading.Thread.__init__(self) self.queue = queue self.auto = auto self.servicesSha = servicesSha self.config = config self.netsrv = NatlasNetworkServices(self.config) def execute_scan(self, work_item): target_data = work_item.target_data utils.create_data_dir(target_data['scan_id']) # setting this here ensures the finally block won't error if we don't submit data response = False try: result = scan(target_data, self.config) if not result: logger.warning("Not submitting data for %s" % target_data['target']) return result.scan_stop() response = self.netsrv.submit_results(result) finally: didFail = response is False utils.cleanup_files(target_data['scan_id'], failed=didFail, saveFails=self.config.save_fails) def run(self): try: while True: work_item = self.get_work() if not work_item: break self.execute_scan(work_item) work_item.complete() except Exception as e: logger.warning("Failed to process work item: %s" % e) capture_exception(e) def get_work(self): # If we're in auto mode, the threads handle getting work from the server if self.auto: target_data = self.netsrv.get_work() # We hit this if we hit an error that we shouldn't recover from. # Primarily version mismatch, at this point. if not target_data: return None if target_data["services_hash"] != self.servicesSha: self.servicesSha = self.netsrv.get_services_file() if not self.servicesSha: logger.warning("Failed to get updated services from %s" % self.config.server) return ScanWorkItem(target_data) else: # Manual target_data = self.queue.get() if not target_data: return None logger.info("Manual Target: %s" % target_data["target"]) return ManualScanWorkItem(self.queue, target_data)
class ThreadScan(threading.Thread): def __init__(self, queue, config, auto=False, servicesSha=''): threading.Thread.__init__(self) self.queue = queue self.auto = auto self.servicesSha = servicesSha self.config = config self.netsrv = NatlasNetworkServices(self.config) def run(self): # If we're in auto mode, the threads handle getting work from the server if self.auto: while True: target_data = self.netsrv.get_work() # We hit this if we hit an error that we shouldn't recover from. # Primarily version mismatch, at this point. if not target_data: os._exit(400) if target_data and target_data[ "services_hash"] != self.servicesSha: self.servicesSha = self.netsrv.get_services_file() if not self.servicesSha: logger.warning( "Failed to get updated services from %s" % config.server) utils.create_data_dir(target_data['scan_id']) result = scan(target_data, self.config) if not result: logger.warning("Not submitting data for %s" % target_data['target']) continue result.scan_stop() response = self.netsrv.submit_results(result) if response is False: utils.cleanup_files(target_data['scan_id'], failed=True, saveFails=self.config.save_fails) else: utils.cleanup_files(target_data['scan_id']) else: #If we're not in auto mode, then the queue is populated with work from local data while True: logger.info("Fetching work from queue") target_data = self.queue.get() if target_data is None: break logger.info("Manual Target: %s" % target_data["target"]) utils.create_data_dir(target_data['scan_id']) result = scan(target_data, self.config) if not result: logger.warning("Not submitting data for %s" % target_data['target']) continue result.scan_stop() response = self.netsrv.submit_results(result) if response is False: utils.cleanup_files(target_data['scan_id'], failed=True, saveFails=self.config.save_fails) else: utils.cleanup_files(target_data['scan_id']) self.queue.task_done()
from natlas.threadscan import ThreadScan from natlas.net import NatlasNetworkServices ERR = { "INVALIDTARGET": 1, "SCANTIMEOUT": 2, "DATANOTFOUND": 3, "INVALIDDATA": 4 } config = Config() MAX_QUEUE_SIZE = (config.max_threads ) # only queue enough work for each of our active threads global_logger = logging.get_logger("MainThread") netsrv = NatlasNetworkServices(config) def add_targets_to_queue(target, q): targetNetwork = ipaddress.ip_network(target.strip()) if targetNetwork.num_addresses == 1: target_data = netsrv.get_work( target=str(targetNetwork.network_address)) if not target_data: return q.put(target_data) else: # Iterate over usable hosts in target, queue.put will block until a queue slot is available for t in targetNetwork.hosts(): target_data = netsrv.get_work(target=str(t)) if not target_data:
class ThreadScan(threading.Thread): def __init__(self, queue, config, auto=False, servicesSha=""): threading.Thread.__init__(self) self.queue = queue self.auto = auto self.servicesSha = servicesSha self.config = config self.netsrv = NatlasNetworkServices(self.config) def execute_scan(self, work_item): target_data = work_item.target_data utils.create_scan_dir(target_data["scan_id"]) # setting this here ensures the finally block won't error if we don't submit data response = False try: result = scan(target_data, self.config) if not result: logger.warning( f"Not submitting data for {target_data['target']}") return result.scan_stop() response = self.netsrv.submit_results(result) finally: didFail = response is False utils.cleanup_files(target_data["scan_id"], failed=didFail, saveFails=self.config.save_fails) def run(self): while True: with push_scope() as scope: add_breadcrumb(category="scan_workflow", message="Fetching work", level="info") work_item = self.get_work() if not work_item: break scope.set_extra("natlas_scan_id", work_item.target_data["scan_id"]) add_breadcrumb( category="scan_workflow", message="Starting scan for %s" % work_item.target_data["target"], level="info", ) try: self.execute_scan(work_item) add_breadcrumb( category="scan_workflow", message="Scan completed. Reporting", level="info", ) work_item.complete() except Exception as e: event_id = capture_exception(e) logger.warning( f"Failed to process work item: {e}. Sentry event id: {event_id}" ) def get_work(self): # If we're in auto mode, the threads handle getting work from the server if self.auto: target_data = self.netsrv.get_work() # We hit this if we hit an error that we shouldn't recover from. # Primarily version mismatch, at this point. if not target_data: return None if target_data["services_hash"] != self.servicesSha: self.servicesSha = self.netsrv.get_services_file() if not self.servicesSha: logger.warning( f"Failed to get updated services from {self.config.server}" ) return ScanWorkItem(target_data) else: # Manual target_data = self.queue.get() if not target_data: return None logger.info(f"Manual Target: {target_data['target']}") return ManualScanWorkItem(self.queue, target_data)