def __init__(self, config=None, db_name='report_tracker.db', purge=False, verbose=None, debug=False, username=None, password=None, profile='nessus'): self.CONFIG_SECTION = profile super(vulnWhispererNessus, self).__init__(config=config) self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True self.purge = purge if config is not None: try: self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') self.nessus_trash = self.config.getbool( self.CONFIG_SECTION, 'trash') try: self.vprint( '{info} Attempting to connect to nessus...'.format( info=bcolors.INFO)) self.nessus = \ NessusAPI(hostname=self.hostname, port=self.nessus_port, username=self.username, password=self.password) self.nessus_connect = True self.vprint( '{success} Connected to nessus on {host}:{port}'. format(success=bcolors.SUCCESS, host=self.hostname, port=str(self.nessus_port))) except Exception as e: self.vprint(e) raise Exception( '{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}' .format(config=self.config.config_in, fail=bcolors.FAIL, e=e)) except Exception as e: self.vprint( '{fail} Could not properly load your config!\nReason: {e}'. format(fail=bcolors.FAIL, e=e)) sys.exit(1)
def __init__( self, config=None, db_name='report_tracker.db', purge=False, verbose=None, debug=False, username=None, password=None, profile='nessus' ): self.CONFIG_SECTION=profile super(vulnWhispererNessus, self).__init__(config=config) self.logger = logging.getLogger('vulnWhispererNessus') if debug: self.logger.setLevel(logging.DEBUG) self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True self.purge = purge if config is not None: try: self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, 'trash') try: self.logger.info('Attempting to connect to nessus...') self.nessus = \ NessusAPI(hostname=self.hostname, port=self.nessus_port, username=self.username, password=self.password) self.nessus_connect = True self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname, port=str(self.nessus_port))) except Exception as e: self.logger.error('Exception: {}'.format(str(e))) raise Exception( 'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( config=self.config.config_in, e=e)) except Exception as e: self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e)) sys.exit(1)
class vulnWhispererNessus(vulnWhispererBase): CONFIG_SECTION = 'nessus' def __init__( self, config=None, db_name='report_tracker.db', purge=False, verbose=None, debug=False, username=None, password=None, ): super(vulnWhispererNessus, self).__init__(config=config) self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True self.purge = purge if config is not None: try: self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, 'trash') try: self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO)) self.nessus = \ NessusAPI(hostname=self.hostname, port=self.nessus_port, username=self.username, password=self.password) self.nessus_connect = True self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS, host=self.hostname, port=str(self.nessus_port))) except Exception as e: self.vprint(e) raise Exception( '{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( config=self.config, fail=bcolors.FAIL, e=e)) except Exception as e: self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, e=e)) sys.exit(0) def scan_count(self, scans, completed=False): """ :param scans: Pulls in available scans :param completed: Only return completed scans :return: """ self.vprint('{info} Gathering all scan data... this may take a while...'.format(info=bcolors.INFO)) scan_records = [] for s in scans: if s: record = {} record['scan_id'] = s['id'] record['scan_name'] = s.get('name', '') record['owner'] = s.get('owner', '') record['creation_date'] = s.get('creation_date', '') record['starttime'] = s.get('starttime', '') record['timezone'] = s.get('timezone', '') record['folder_id'] = s.get('folder_id', '') try: for h in self.nessus.get_scan_history(s['id']): record['uuid'] = h.get('uuid', '') record['status'] = h.get('status', '') record['history_id'] = h.get('history_id', '') record['last_modification_date'] = \ h.get('last_modification_date', '') record['norm_time'] = \ self.nessus.get_utc_from_local(int(record['last_modification_date' ]), local_tz=self.nessus.tz_conv(record['timezone' ])) scan_records.append(record.copy()) except Exception as e: # Generates error each time nonetype is encountered. # print(e) pass if completed: scan_records = [s for s in scan_records if s['status'] == 'completed'] return scan_records def whisper_nessus(self): if self.nessus_connect: scan_data = self.nessus.get_scans() folders = scan_data['folders'] scans = scan_data['scans'] all_scans = self.scan_count(scans) if self.uuids: scan_list = [scan for scan in all_scans if scan['uuid'] not in self.uuids and scan['status'] == 'completed'] else: scan_list = all_scans self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO, new=len(scan_list))) if not scan_list: self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO)) exit(0) # Create scan subfolders for f in folders: if not os.path.exists(self.path_check(f['name'])): if f['name'] == 'Trash' and self.nessus_trash: os.makedirs(self.path_check(f['name'])) elif f['name'] != 'Trash': os.makedirs(self.path_check(f['name'])) else: os.path.exists(self.path_check(f['name'])) self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format( scan=self.path_check(f['name' ]), info=bcolors.INFO)) # try download and save scans into each folder the belong to scan_count = 0 # TODO Rewrite this part to go through the scans that have aleady been processed for s in scan_list: scan_count += 1 ( scan_name, scan_id, history_id, norm_time, status, uuid, ) = ( s['scan_name'], s['scan_id'], s['history_id'], s['norm_time'], s['status'], s['uuid'], ) # TODO Create directory sync function which scans the directory for files that exist already and populates the database folder_id = s['folder_id'] scan_history = self.nessus.get_scan_history(scan_id) folder_name = next(f['name'] for f in folders if f['id' ] == folder_id) if status == 'completed': file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id, history_id, norm_time, 'csv') repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_')) file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) relative_path_name = self.path_check(folder_name + '/' + file_name) if os.path.isfile(relative_path_name): if self.develop: csv_in = pd.read_csv(relative_path_name) record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), csv_in.shape[0], self.CONFIG_SECTION, uuid, 1, ) self.record_insert(record_meta) self.vprint( '{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name)) else: file_req = \ self.nessus.download_scan(scan_id=scan_id, history=history_id, export_format='csv') clean_csv = \ pd.read_csv(io.StringIO(file_req.decode('utf-8' ))) if len(clean_csv) > 2: self.vprint('Processing %s/%s for scan: %s' % (scan_count, len(scan_list), scan_name)) columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output'] for col in columns_to_cleanse: clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser) clean_csv['Synopsis'] = \ clean_csv['Description' ].astype(str).apply(self.cleanser) clean_csv.to_csv(relative_path_name, index=False) record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), clean_csv.shape[0], self.CONFIG_SECTION, uuid, 1, ) self.record_insert(record_meta) self.vprint('{info} {filename} records written to {path} '.format(info=bcolors.INFO, filename=clean_csv.shape[ 0], path=file_name)) else: record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), clean_csv.shape[0], self.CONFIG_SECTION, uuid, 1, ) self.record_insert(record_meta) self.vprint(file_name + ' has no host available... Updating database and skipping!' ) self.conn.close() '{success} Scan aggregation complete! Connection to database closed.'.format(success=bcolors.SUCCESS) else: self.vprint('{fail} Failed to use scanner at {host}'.format(fail=bcolors.FAIL, host=self.hostname + ':' + self.nessus_port))
def __init__(self, config=None, db_name='report_tracker.db', purge=False, verbose=None, debug=False): self.verbose = verbose self.nessus_connect = False self.develop = True self.purge = purge if config is not None: try: self.config = vwConfig(config_in=config) self.nessus_enabled = self.config.getbool('nessus', 'enabled') if self.nessus_enabled: self.nessus_hostname = self.config.get('nessus', 'hostname') self.nessus_port = self.config.get('nessus', 'port') self.nessus_username = self.config.get('nessus', 'username') self.nessus_password = self.config.get('nessus', 'password') self.nessus_writepath = self.config.get('nessus', 'write_path') self.nessus_dbpath = self.config.get('nessus', 'db_path') self.nessus_trash = self.config.getbool('nessus', 'trash') self.verbose = self.config.getbool('nessus', 'verbose') try: self.vprint( '{info} Attempting to connect to nessus...'.format(info=bcolors.INFO)) self.nessus = NessusAPI(hostname=self.nessus_hostname, port=self.nessus_port, username=self.nessus_username, password=self.nessus_password) self.nessus_connect = True self.vprint( '{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS, host=self.nessus_hostname, port=str(self.nessus_port))) except Exception as e: self.vprint(e) raise Exception( "{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}".format(config=self.config, fail=bcolors.FAIL, e=e)) except Exception as e: self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, e=e)) sys.exit(0) if db_name is not None: if self.nessus_dbpath: self.database = os.path.join(self.nessus_dbpath, db_name) else: self.database = os.path.abspath(os.path.join(os.path.dirname( __file__ ), 'database', db_name)) try: self.conn = sqlite3.connect(self.database) self.cur = self.conn.cursor() self.vprint("{info} Connected to database at {loc}".format(info=bcolors.INFO, loc=self.database)) except Exception as e: self.vprint("{fail} Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist".format(e=e, fail=bcolors.FAIL, loc=self.database)) else: self.vprint('{fail} Please specify a database to connect to!'.format(fail=bcolors.FAIL)) exit(0) self.table_columns = ['scan_name', 'scan_id', 'last_modified', 'filename', 'download_time', 'record_count', 'source', 'uuid', 'processed'] self.init() self.uuids = self.retrieve_uuids() self.processed = 0 self.skipped = 0 self.scan_list = []
class vulnWhisperer(object): def __init__(self, config=None, db_name='report_tracker.db', purge=False, verbose=None, debug=False): self.verbose = verbose self.nessus_connect = False self.develop = True self.purge = purge if config is not None: try: self.config = vwConfig(config_in=config) self.nessus_enabled = self.config.getbool('nessus', 'enabled') if self.nessus_enabled: self.nessus_hostname = self.config.get('nessus', 'hostname') self.nessus_port = self.config.get('nessus', 'port') self.nessus_username = self.config.get('nessus', 'username') self.nessus_password = self.config.get('nessus', 'password') self.nessus_writepath = self.config.get('nessus', 'write_path') self.nessus_dbpath = self.config.get('nessus', 'db_path') self.nessus_trash = self.config.getbool('nessus', 'trash') self.verbose = self.config.getbool('nessus', 'verbose') try: self.vprint( '{info} Attempting to connect to nessus...'.format(info=bcolors.INFO)) self.nessus = NessusAPI(hostname=self.nessus_hostname, port=self.nessus_port, username=self.nessus_username, password=self.nessus_password) self.nessus_connect = True self.vprint( '{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS, host=self.nessus_hostname, port=str(self.nessus_port))) except Exception as e: self.vprint(e) raise Exception( "{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}".format(config=self.config, fail=bcolors.FAIL, e=e)) except Exception as e: self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, e=e)) sys.exit(0) if db_name is not None: if self.nessus_dbpath: self.database = os.path.join(self.nessus_dbpath, db_name) else: self.database = os.path.abspath(os.path.join(os.path.dirname( __file__ ), 'database', db_name)) try: self.conn = sqlite3.connect(self.database) self.cur = self.conn.cursor() self.vprint("{info} Connected to database at {loc}".format(info=bcolors.INFO, loc=self.database)) except Exception as e: self.vprint("{fail} Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist".format(e=e, fail=bcolors.FAIL, loc=self.database)) else: self.vprint('{fail} Please specify a database to connect to!'.format(fail=bcolors.FAIL)) exit(0) self.table_columns = ['scan_name', 'scan_id', 'last_modified', 'filename', 'download_time', 'record_count', 'source', 'uuid', 'processed'] self.init() self.uuids = self.retrieve_uuids() self.processed = 0 self.skipped = 0 self.scan_list = [] def vprint(self, msg): if self.verbose: print(msg) def create_table(self): self.cur.execute("create table if not exists scan_history (id integer primary key, scan_name text, scan_id integer, last_modified date, filename text, download_time date, record_count integer, source text, uuid text, processed integer)") self.conn.commit() def delete_table(self): self.cur.execute('drop table if exists scan_history') self.conn.commit() def init(self): if self.purge: self.delete_table() self.create_table() def cleanser(self, _data): repls = ('\n', '|||'), ('\r', '|||'), (',',';') data = reduce(lambda a, kv: a.replace(*kv), repls, _data) return data def path_check(self, _data): if self.nessus_writepath: data = self.nessus_writepath + '/' + _data return data def scan_count(self, scans, completed=False): """ :param scans: Pulls in available scans :param completed: Only return completed scans :return: """ self.vprint('{info} Gathering all scan data... this may take a while...'.format(info=bcolors.INFO)) scan_records = [] for s in scans: if s: record = {} record['scan_id'] = s['id'] record['scan_name'] = s.get('name', '') record['owner'] = s.get('owner', '') record['creation_date'] = s.get('creation_date', '') record['starttime'] = s.get('starttime', '') record['timezone'] = s.get('timezone', '') record['folder_id'] = s.get('folder_id', '') try: for h in self.nessus.get_scan_history(s['id']): record['uuid'] = h.get('uuid', '') record['status'] = h.get('status', '') record['history_id'] = h.get('history_id', '') record['last_modification_date'] = h.get('last_modification_date', '') record['norm_time'] = self.nessus.get_utc_from_local(int(record['last_modification_date']), local_tz=self.nessus.tz_conv(record['timezone'])) scan_records.append(record.copy()) except Exception as e: print(e) pass if completed: scan_records = [s for s in scan_records if s['status'] == 'completed'] return scan_records def record_insert(self, record): self.cur.execute("insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)".format( table_columns=', '.join(self.table_columns)), record) self.conn.commit() def retrieve_uuids(self): """ Retrieves UUIDs from database and checks list to determine which files need to be processed. :return: """ self.conn.text_factory = str self.cur.execute('select uuid from scan_history') results = frozenset([r[0] for r in self.cur.fetchall()]) return results def whisper_nessus(self): if self.nessus_connect: scan_data = self.nessus.get_scans() folders = scan_data['folders'] scans = scan_data['scans'] all_scans = self.scan_count(scans) if self.uuids: scan_list = [scan for scan in all_scans if (scan['uuid'] not in self.uuids and scan['status']=='completed')] else: scan_list = all_scans self.vprint("{info} Identified {new} scans to be processed".format(info=bcolors.INFO, new=len(scan_list))) if not scan_list: self.vprint("{info} No new scans to process. Exiting...".format(info=bcolors.INFO)) exit(0) # Create scan subfolders for f in folders: if not os.path.exists(self.path_check(f['name'])): if f['name'] == 'Trash' and self.nessus_trash: os.makedirs(self.path_check(f['name'])) elif f['name'] != 'Trash': os.makedirs(self.path_check(f['name'])) else: os.path.exists(self.path_check(f['name'])) self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format( scan=self.path_check(f['name']), info=bcolors.INFO)) # try download and save scans into each folder the belong to scan_count = 0 # TODO Rewrite this part to go through the scans that have aleady been processed for s in scan_list: scan_count += 1 scan_name, scan_id, history_id,\ norm_time, status, uuid = s['scan_name'], s['scan_id'], s['history_id'],\ s['norm_time'], s['status'], s['uuid'] # TODO Create directory sync function which scans the directory for files that exist already and populates the database folder_id = s['folder_id'] scan_history = self.nessus.get_scan_history(scan_id) folder_name = next(f['name'] for f in folders if f['id'] == folder_id) if status == 'completed': file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id, history_id, norm_time, 'csv') repls = ('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_') file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) relative_path_name = self.path_check(folder_name + '/' + file_name) if os.path.isfile(relative_path_name): if self.develop: csv_in = pd.read_csv(relative_path_name) record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), csv_in.shape[0], 'nessus', uuid, 1) self.record_insert(record_meta) self.vprint( "{info} File {filename} already exist! Updating database".format(info=bcolors.INFO, filename=relative_path_name)) else: file_req = self.nessus.download_scan(scan_id=scan_id, history=history_id, export_format='csv') clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8'))) if len(clean_csv) > 2: self.vprint("Processing %s/%s for scan: %s" % (scan_count, len(scan_history), scan_name)) clean_csv['CVSS'] = clean_csv['CVSS'].astype(str).apply(self.cleanser) clean_csv['CVE'] = clean_csv['CVE'].astype(str).apply(self.cleanser) clean_csv['Description'] = clean_csv['Description'].astype(str).apply(self.cleanser) clean_csv['Synopsis'] = clean_csv['Description'].astype(str).apply(self.cleanser) clean_csv['Solution'] = clean_csv['Solution'].astype(str).apply(self.cleanser) clean_csv['See Also'] = clean_csv['See Also'].astype(str).apply(self.cleanser) clean_csv['Plugin Output'] = clean_csv['Plugin Output'].astype(str).apply(self.cleanser) clean_csv.to_csv(relative_path_name, index=False) record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), clean_csv.shape[0], 'nessus', uuid, 1) self.record_insert(record_meta) self.vprint("{info} {filename} records written to {path} ".format(info=bcolors.INFO, filename=clean_csv.shape[0], path=file_name)) else: record_meta = ( scan_name, scan_id, norm_time, file_name, time.time(), clean_csv.shape[0], 'nessus', uuid, 1) self.record_insert(record_meta) self.vprint(file_name + ' has no host available... Updating database and skipping!') self.conn.close() "{success} Scan aggregation complete! Connection to database closed.".format(success=bcolors.SUCCESS) else: self.vprint('{fail} Failed to use scanner at {host}'.format(fail=bcolors.FAIL, host=self.nessus_hostname+':'+self.nessus_port))