def test_database_update_schema(self,log): db = database.Database() db.create_db_dir() db.create_db() util.run_db_scripts(self.gci) db.update_schema() validator = datavalidator.DataValidator() schema = validator.get_schema() self.assertTrue(schema['test_http'][5][1] == 'command') self.assertTrue(schema['test_http2'][6][1] == 'path') self.assertTrue(len(schema['test_telnet']) == 7) # set global config instance to the differing column config file self.gci = GlobalConfig(self.plugins_config_diff_file,self.global_config_file,True) self.gci.read_global_config() self.gci.read_plugin_config() db2 = database.Database() db2.update_schema() validator2 = datavalidator.DataValidator() schema2 = validator2.get_schema() self.assertTrue(schema2['test_http'][5][1] == 'unit_test_data_1') self.assertTrue(schema2['test_http2'][6][1] == 'unit_test_data_2') self.assertTrue(len(schema2['test_telnet']) == 8) self.assertTrue(schema2['test_telnet'][7][1] == 'unit_test_data_3') # set global config instance back to normal self.gci = GlobalConfig(self.plugins_config_file,self.global_config_file,True) self.gci.read_global_config() self.gci.read_plugin_config() shutil.rmtree(os.getcwd() + self.test_db_dir)
def construct_port_summary_list(self, rqst, uom, units): g_config = GlobalConfig() plugins_dictionary = g_config.get_plugin_dictionary() json_list = [] for key, val in plugins_dictionary.items(): json_list.append(self.construct_port_summary(rqst, val['port'], val['table'], uom, units)) return json_list
class Table_Insert_test(unittest.TestCase): def setUp(self): self.test_db_dir = '/tests/database/test_database' self.test_db_file = '/tests/database/test_database/honeyDB.sqlite' # test configuration files self.plugins_config_file = 'tests/database/test_config/plugins.cfg' self.plugins_config_diff_file = 'tests/database/test_config/plugins_diff.cfg' self.plugins_config_diff_table_file = 'tests/database/test_config/plugins_diff_table.cfg' self.global_config_file = 'tests/database/test_config/global.cfg' # create global config instance self.gci = GlobalConfig(self.plugins_config_file,self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config() self.telnet_data = {'test_telnet':{'session':'abcdefghijklmnop','eventDateTime':'01-02-2016 11:22:33.123', 'peerAddress':'24.33.21.123', 'localAddress':'192.168.0.55', 'input_type':'a string','user_input':'another string'}} self.telnet_data_noip = {'test_telnet':{'session':'abcdefghijklmnop','eventDateTime':'01-02-2016 11:22:33.123', 'input_type':'a string','user_input':'another string'}} @patch.object(Logger,'__new__') def test_telnet_all_values(self,log): db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() Table_Insert.prepare_data_for_insertion(validator.get_schema(), self.telnet_data) connection = sqlite3.connect(self.gci['Database']['path']) cursor = connection.cursor() row = cursor.execute('select * from test_telnet;').fetchall()[0] check_list = self.helper_get_values_out(self.telnet_data) self.assertTrue(set(row) > set(check_list)) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_telnet_missing_non_required_values(self,log): db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() Table_Insert.prepare_data_for_insertion(validator.get_schema(), self.telnet_data_noip) connection = sqlite3.connect(self.gci['Database']['path']) cursor = connection.cursor() row = cursor.execute('select * from test_telnet;').fetchall()[0] check_list = self.helper_get_values_out(self.telnet_data_noip) bad_check_list = self.helper_get_values_out(self.telnet_data) self.assertTrue(set(row) > set(check_list)) self.assertFalse(set(row) > set(bad_check_list)) shutil.rmtree(os.getcwd() + self.test_db_dir) def helper_get_values_out(self,dictionary): inner_dict = dictionary[util.get_first_key_value_of_dictionary(dictionary)] list = [] for item in inner_dict: list.append(inner_dict[item]) return list
class PortManager: # Port Manager: calls necessary managers and utilities to generate parameters for sql. # List of valid ports it can receive is taken from the Configuration setup. # validPortNumbers = () def __init__(self): self.g_config = GlobalConfig() self.validPortNumbers = self.g_config.get_ports() self.date_time_field = self.g_config.get_db_datetime_name() self.log = Logger().get('reportserver.manager.PortManager.PortManager') def isPortValid(self, port_number): if (port_number in self.validPortNumbers): return True else: return False def getPort(self, port_number, uom, unit): self.log.info("Retrieving port:" + str(port_number) + "uom:" + uom + " size: " + str(unit)) items = [] if self.isPortValid(port_number): results = DatabaseHandler().get_json_by_time(port_number, uom, unit) items = utilities.process_data(results) port_json = { 'port': str(port_number), 'timespan': uom + "=" + str(unit), 'items':items } return port_json def get_port_attack_count(self, tablename, unit, uom): fromDate = dateTimeUtility.get_begin_date_iso(unit, uom) sql = "select count(distinct session) as total_attacks from %s where %s >= '%s' " %(tablename, self.date_time_field, fromDate) self.log.debug("sql is:" + sql) result = DatabaseHandler().query_db(sql)[0] return int(result['total_attacks']) def get_unique_ips(self, tablename, unit, uom): fromDate = dateTimeUtility.get_begin_date_iso(unit, uom) sql = "select count(distinct peerAddress) as unique_ips from %s where %s >= '%s' " % (tablename, self.date_time_field, fromDate) self.log.debug("sql is:" + sql) result = DatabaseHandler().query_db(sql)[0] return int(result['unique_ips'])
class DatabaseHandler: def __init__(self): self.global_config = GlobalConfig() self.db_path = self.global_config['Database']['path'] self.log = Logger().get('reportserver.dao.DatabaseHandler.DatabaseHandler') # Connect to given database. # Defaults to the honeypot db, but another path can be passed in (mainly for testing). # Database needs to exist first. def connect(self, database_name): if (database_name == None): database_name = self.db_path if not os.path.exists(database_name): self.log.error("Database does not exist in path: " + database_name) return None try: conn = sqlite3.connect(database_name) except sqlite3.OperationalError as oe: self.log.error("****Problem connecting to database*** at: " + database_name) self.log.error(oe) else: return conn # Query DB and return JSON def query_db(self, query, args=(), one=False, db=None): #print ("#debug args are: " +str(args)) cur = self.connect(db).cursor() cur.execute(query, args) r = [dict((cur.description[i][0], value) \ for i, value in enumerate(row)) for row in cur.fetchall()] cur.connection.close() return (r[0] if r else None) if one else r # Unit of Measure could be "weeks", "days", "hours", "minutes". # Return all data from the DB within that measure of time as JSON. def get_json_by_time(self, portnumber, uom, units): begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units) tableName = self.global_config.get_plugin_config(portnumber)['table'] date_time_field = self.global_config.get_db_datetime_name() # query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso)) queryString = "SELECT * FROM %s where %s >= '%s' order by id, %s" % (tableName, date_time_field, begin_date_iso, date_time_field) #args = (tableName, date_time_field, begin_date_iso) self.log.info("queryString is: " + str(queryString)) #print ("args to use: " + str(args)) results = self.query_db(queryString) self.log.debug("results: " + str(results)) return results
class IpsManager: # Ips Manager: calls necessary managers and utilities to generate parameters for sql. # validPortNumbers = () def __init__(self): self.g_config = GlobalConfig() self.valid_port_numbers = self.g_config.get_ports() self.date_time_field = self.g_config.get_db_datetime_name() self.log = Logger().get('reportserver.manager.IpsManager.py') def get_data(self, ipaddress, uom, unit): self.log.info("Retrieving ipaddress data: " + str(ipaddress) + " uom: " + uom + " size: " + str(unit)) port_data = [] for port in self.valid_port_numbers: results = self.get_json_by_ip(port, ipaddress, uom, unit) items = utilities.process_data(results) port_data.append({port:items}) port_json = { 'ipaddress': str(ipaddress), 'timespan': uom + "=" + str(unit), 'ports':port_data } return port_json def get_json_by_ip(self, portnumber, ipaddress, uom, units): begin_date_iso = dateTimeUtility.get_begin_date_iso(uom, units) table_name = self.g_config.get_plugin_config(portnumber)['table'] date_time_field = self.g_config.get_db_datetime_name() # query = query_db("SELECT * FROM %s where (datetime > '%s')" % (tableName, query_date_iso)) queryString = "SELECT * FROM %s where %s >= '%s' and peerAddress = '%s' order by id, %s" % ( table_name, date_time_field, begin_date_iso, ipaddress, date_time_field) # args = (tableName, date_time_field, begin_date_iso) self.log.info("queryString is: " + str(queryString)) # print ("args to use: " + str(args)) results = DatabaseHandler().query_db(queryString) self.log.debug("results: " + str(results)) return results
def __init__(self, plugin_cfg_path, global_cfg_path): self._global_config = GlobalConfig(plugin_cfg_path, global_cfg_path) self._plugin_imports = {} self._listener_list= {} self._running_plugins_list = [] self._data_manager = None self._shutting_down = False self._log = None self._pid = os.getpid()
def __init__(self): plugin_cfg_path = os.getenv('RECCE7_PLUGIN_CONFIG') or 'config/plugins.cfg' global_cfg_path = os.getenv('RECCE7_GLOBAL_CONFIG') or 'config/global.cfg' self.g_config = GlobalConfig(plugin_cfg_path, global_cfg_path) self.g_config.read_plugin_config() self.g_config.read_global_config() self.host = self.g_config.get_report_server_host() self.port = self.g_config.get_report_server_port() log_path = self.g_config['ReportServer']['reportserver.logName'] log_level = self.g_config['ReportServer']['reportserver.logLevel'] self.log = Logger(log_path, log_level).get('reportserver.server.SimpleHTTPServer.SimpleHTTPServer')
def setUp(self): self.test_db_dir = "/tests/database/test_database" self.test_db_file = "/tests/database/test_database/honeyDB.sqlite" # test configuration files self.plugins_config_file = "tests/database/test_config/plugins.cfg" self.plugins_config_diff_file = "tests/database/test_config/plugins_diff.cfg" self.plugins_config_diff_table_file = "tests/database/test_config/plugins_diff_table.cfg" self.global_config_file = "tests/database/test_config/global.cfg" # create global config instance self.gci = GlobalConfig(self.plugins_config_file, self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config()
class datamanager_test(unittest.TestCase): def setUp(self): self.test_db_dir = "/tests/database/test_database" self.test_db_file = "/tests/database/test_database/honeyDB.sqlite" # test configuration files self.plugins_config_file = "tests/database/test_config/plugins.cfg" self.plugins_config_diff_file = "tests/database/test_config/plugins_diff.cfg" self.plugins_config_diff_table_file = "tests/database/test_config/plugins_diff_table.cfg" self.global_config_file = "tests/database/test_config/global.cfg" # create global config instance self.gci = GlobalConfig(self.plugins_config_file, self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config() @patch.object(Logger, "__new__") def test_datamanager_init(self, log): dm = datamanager.DataManager() self.assertTrue(os.path.isdir(os.getcwd() + self.test_db_dir)) self.assertTrue(os.path.isfile(os.getcwd() + self.test_db_file)) self.assertIsInstance(dm.q, dataqueue.DataQueue) shutil.rmtree(os.getcwd() + self.test_db_dir)
class SimpleHttpServer: def __init__(self): plugin_cfg_path = os.getenv('RECCE7_PLUGIN_CONFIG') or 'config/plugins.cfg' global_cfg_path = os.getenv('RECCE7_GLOBAL_CONFIG') or 'config/global.cfg' self.g_config = GlobalConfig(plugin_cfg_path, global_cfg_path) self.g_config.read_plugin_config() self.g_config.read_global_config() self.host = self.g_config.get_report_server_host() self.port = self.g_config.get_report_server_port() log_path = self.g_config['ReportServer']['reportserver.logName'] log_level = self.g_config['ReportServer']['reportserver.logLevel'] self.log = Logger(log_path, log_level).get('reportserver.server.SimpleHTTPServer.SimpleHTTPServer') def setupAndStart(self): server_address = (self.host, self.port) request_handler = RestRequestHandler # instantiate a server object httpd = HTTPServer (server_address, request_handler) print(time.asctime(), "Server Starting - %s:%s" % (self.host, self.port)) try: # start serving pages httpd.serve_forever () except KeyboardInterrupt: pass httpd.server_close() print(time.asctime(), "Server Stopped - %s:%s" % (self.host, self.port))
def test_get_json_by_time(self): plugin_cfg_path = os.getenv("RECCE7_PLUGIN_CONFIG") or "config/plugins.cfg" global_cfg_path = os.getenv("RECCE7_GLOBAL_CONFIG") or "config/global.cfg" global_config = GlobalConfig(plugin_cfg_path, global_cfg_path, True) global_config.read_global_config() global_config.read_plugin_config() test_start_date = datetime.datetime(1999, month=12, day=31, hour=23, minute=59, second=59) # successes = 0 # fails = 0 for count in range(0, 2): if count == 0: portnumber = 8082 elif count == 1: portnumber = 8083 else: portnumber = 8023 for x in range(0, 500): d = datetime.timedelta(weeks=x) query_date = test_start_date + d query_date_iso = query_date.isoformat() tableName = global_config.get_plugin_config(portnumber)["table"] query_string = "SELECT * FROM %s where (eventDateTime >= '%s')" % (tableName, query_date_iso) json_query = DatabaseHandler().query_db(query_string, db="TestDB.sqlite") for y in range(0, len(json_query) - 1): date = json_query[y].get("eventDateTime") self.assertGreaterEqual(date, query_date_iso)
class TableCreationTestCase(unittest.TestCase): def setUp(self): self.gci = GlobalConfig('tests/database/test_configs/Table_Init_test.cfg') @patch.object(GlobalConfig, 'get_db_dir', return_value=testpath) def tearDown(self,gci_get_db_dir): os.remove(self.gci.get_db_dir() + '/honeyDB.sqlite') @patch.object(GlobalConfig, 'get_db_dir', return_value=testpath) def test_create_table(self,gci_get_db_dir): database.create_default_database(self.gci) Table_Init.create_table('TestTable1',self.gci) connection = sqlite3.connect(self.gci.get_db_dir() + '/honeyDB.sqlite') cursor = connection.cursor() table_list = [] for table_tuple in cursor.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall(): table_list.append(table_tuple[0]) self.assertIn('TestTable1', table_list) connection.close() @patch.object(GlobalConfig, 'get_db_dir', return_value=testpath) def test_verify_data_types(self,gci_get_db_dir): database.create_default_database(self.gci) good_list = database.create_dict_config_column_list(self.gci).get('test_telnet_test') self.assertIsNone(Table_Init.verify_data_types(good_list)) bad_list = database.create_dict_config_column_list(self.gci).get('test_telnet_test') bad_list[0][2] = 'ASDF' self.assertRaises(ValueError, Table_Init.verify_data_types, bad_list) @patch.object(GlobalConfig, 'get_db_dir', return_value=testpath) def test_check_table_exists(self,gci_get_db_dir): database.create_default_database(self.gci) self.assertTrue(Table_Init.check_table_exists('test_http_test',self.gci)) self.assertTrue(Table_Init.check_table_exists('test_telnet_test',self.gci)) self.assertFalse(Table_Init.check_table_exists('TESTTABLE1',self.gci)) self.assertFalse(Table_Init.check_table_exists('TestTable2',self.gci))
def setUp(self): # Testing for correct DB plugin_cfg_path = "tests/reportserver/testconfig/plugins.cfg" global_cfg_path = "tests/reportserver/testconfig/global.cfg" global_config = GlobalConfig(plugin_cfg_path, global_cfg_path, True) global_config.read_global_config() global_config.read_plugin_config() conn = sqlite3.connect("TestDB.sqlite") c = conn.cursor() c.execute("""CREATE TABLE test_http (port int, data text, eventDateTime text)""") c.execute("""CREATE TABLE test_http2 (port int, data text, eventDateTime text)""") c.execute("""CREATE TABLE test_telnet (port int, data text, eventDateTime text)""") test_start_date = datetime.datetime(1999, month=12, day=31, hour=23, minute=59, second=59) for x in range(0, 500): d = datetime.timedelta(weeks=x) insert_date = test_start_date + d insert_date_iso = insert_date.isoformat() c.execute("INSERT INTO test_http VALUES (8082,'TEXT','%s')" % insert_date_iso) c.execute("INSERT INTO test_http2 VALUES (8083,'TEXT','%s')" % insert_date_iso) c.execute("INSERT INTO test_telnet VALUES (8023,'TEXT','%s')" % insert_date_iso) conn.commit()
def test_connect(self): # Negative testing self.assertIsNone(DatabaseHandler().connect("database")) self.assertIsNone(DatabaseHandler().connect("database.db")) self.assertIsNone(DatabaseHandler().connect("asdl;kfjeiei")) self.assertIsNone(DatabaseHandler().connect("./honeyDB/honeyDB.sqllite")) self.assertIsNone(DatabaseHandler().connect("./honeyDB/honeyDB.db")) self.assertIsNone(DatabaseHandler().connect(" ")) self.assertIsNone(DatabaseHandler().connect("")) # Testing for correct DB plugin_cfg_path = "tests/reportserver/testconfig/plugins.cfg" global_cfg_path = "tests/reportserver/testconfig/global.cfg" global_config = GlobalConfig(plugin_cfg_path, global_cfg_path, True) global_config.read_global_config() global_config.read_plugin_config() db = Database() db.create_db_dir() db.create_db() db_path = global_config["Database"]["path"] self.assertTrue(sqlite3.connect(db_path)) self.assertTrue(DatabaseHandler().connect(db_path)) self.assertTrue(DatabaseHandler().connect(None))
def setUp(self): self.test_db_dir = '/tests/database/test_database' self.test_db_file = '/tests/database/test_database/honeyDB.sqlite' # test configuration files self.plugins_config_file = 'tests/database/test_config/plugins.cfg' self.plugins_config_diff_file = 'tests/database/test_config/plugins_diff.cfg' self.plugins_config_diff_table_file = 'tests/database/test_config/plugins_diff_table.cfg' self.global_config_file = 'tests/database/test_config/global.cfg' # create global config instance self.gci = GlobalConfig(self.plugins_config_file,self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config() self.telnet_data = {'test_telnet':{'session':'abcdefghijklmnop','eventDateTime':'01-02-2016 11:22:33.123', 'peerAddress':'24.33.21.123', 'localAddress':'192.168.0.55', 'input_type':'a string','user_input':'another string'}} self.telnet_data_noip = {'test_telnet':{'session':'abcdefghijklmnop','eventDateTime':'01-02-2016 11:22:33.123', 'input_type':'a string','user_input':'another string'}}
class WorldmapServiceHandler(): def __init__(self): self.log = Logger().get('reportserver.manager.WorldmapServiceManager.py') self.global_config = GlobalConfig() self.global_config.read_plugin_config() self.global_config.read_global_config() def process(self, rqst, path_tokens, query_tokens): global have_basemap if not have_basemap: err_msg = \ ('<html><head><title>WorldMap</title></head><body>' 'To enable WorldMap generation, please visit ' '<a href="https://recce7.github.io/">the documentation</a> and ' 'follow the directions for installing the Basemap library.' '</body></html>') rqst.send_response(200) #todo make this configurable for allow-origin rqst.send_header("Access-Control-Allow-Origin","http://localhost:8000") rqst.send_header('Content-Type', 'text/html') rqst.send_header('Content-Length', len(err_msg)) rqst.end_headers() rqst.flush_headers() rqst.wfile.write(bytes(err_msg, "utf-8")) rqst.wfile.flush() return uom = None units = None self.log.info("processing ipaddress request:" + str(path_tokens) + str(query_tokens)) try: time_period = utilities.validate_time_period(query_tokens) uom = time_period[0] units = time_period[1] except ValueError: rqst.badRequest(units) return if len(path_tokens) >= 5: rqst.badRequest() return else: self.construct_worldmap(rqst, uom, units) def construct_worldmap(self, rqst, uom, units): #call to construct port list #find unique ips by port #merge the results togoether #build the map #probably want to look at the PortsServiceHandler.py or IpsServiceHandler.py to follow those patterns. ip_map = pickle.loads(pickle_bytes) pts = self.get_point_list(uom, units) for pt in pts: srclat, srclong = pt x, y = ip_map(srclong, srclat) plt.plot(x, y, 'o', color='#ff0000', ms=2.7, markeredgewidth=1.0) plt.savefig('reportserver/worldmap.png', dpi=600) img = Image.open('reportserver/worldmap.png') draw = ImageDraw.Draw(img) font = ImageFont.truetype( "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 175) draw.text((50, 50), "Unique IP addresses: last %s %s" % (units, uom), (0, 0, 0), font=font) font = ImageFont.truetype( "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 125) draw.text((50, 325), "Total: %s" % (len(pts)), (0, 0, 0), font=font) # draw = ImageDraw.Draw(img) # draw = ImageDraw.Draw(img) img.save("reportserver/worldmap.png") rqst.sendPngResponse("reportserver/worldmap.png", 200) def get_point_list(self, uom, units): begin_date = dateTimeUtility.get_begin_date_iso(uom, units) query_string = ('select lat,long ' 'from (' 'select distinct lat,long,timestamp, ip ' 'from ipInfo ' 'where lat is not null ' 'and long is not null ' 'and datetime(timestamp) > datetime(\'' + begin_date + '\')' ');') connection = sqlite3.connect(self.global_config['Database']['path']) cursor = connection.cursor() return cursor.execute(query_string).fetchall()
def __init__(self): self.global_config = GlobalConfig() self.log = Logger().get('database.database.Database')
def __init__(self): self.global_config = GlobalConfig() self.db_path = self.global_config['Database']['path'] self.log = Logger().get('reportserver.dao.DatabaseHandler.DatabaseHandler')
class ApplicationUpdater(): def __init__(self): self.globalconfig = GlobalConfig() self.nodes = self.globalconfig.get_hosts() self.notifications = Notifications() self.application = Applications() self.logger = logging.getLogger(__name__) def process_app(self, app, number_per_node = 1, number_of_containers = 1): self.nodes = self.globalconfig.get_hosts() app_details = None if not self.application.set_application_lock(app): return False else: app_details = self.application.get(app, containers=False) if app_details['command'] == "": self.logger.debug("Command not set, application skipped") return False if app_details['docker_image'] == "": self.logger.debug("Docker Image not set, application skipped") return False number_per_node = 1 number_of_containers = 1 if "container_count" in app_details: try: if app_details['container_count'] == "all": number_of_containers = len(self.nodes) else: number_of_containers = float(app_details['container_count']) try: number_per_node = number_of_containers / len(self.nodes) except ZeroDivisionError: self.logger.info("No nodes, setting number_per_node to 0") number_per_node = 0 except ValueError: pass try: app_class = DockerFunctions(app, self.nodes, self.notifications) ss = Scheduler(app_class.start_instance, app_class.shutdown_instance, app_class.list_nodes, app_class.health_check, number_per_node) self.logger.debug("Starting scheduler for app:{}".format(app)) output = list() for event in ss.run(number_of_containers): self.logger.debug("Scheduler Event: {}".format(event)) output.append(event) self.logger.debug(output) self.logger.debug("Finished scheduler for app: {}".format(app)) if ss.success == True: self.application.set_application_state(app, "RUNNING") else: self.application.set_application_state(app, "ERROR: {}".format(output)) except Exception as e: self.logger.info("======= Scheduler Failed: {}".format(e.message)) self.application.set_application_state(app, e.message) self.application.remove_application_lock(app) return True
class datavalidator_test(unittest.TestCase): def setUp(self): self.test_db_dir = '/tests/database/test_database' self.test_db_file = '/tests/database/test_database/honeyDB.sqlite' # test configuration files self.plugins_config_file = 'tests/database/test_config/plugins.cfg' self.plugins_config_diff_file = 'tests/database/test_config/plugins_diff.cfg' self.plugins_config_diff_table_file = 'tests/database/test_config/plugins_diff_table.cfg' self.global_config_file = 'tests/database/test_config/global.cfg' # create global config instance self.gci = GlobalConfig(self.plugins_config_file,self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config() self.tables_test = ['p0f', 'ipInfo', 'sessions', 'test_http', 'test_http2', 'test_telnet'] self.table_schema_test = {'sessions': [(0, 'session', 'TEXT', 0, None, 1), (1, 'table_name', 'TEXT', 1, None, 2)], 'test_http2': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'command', 'TEXT', 0, None, 0), (6, 'path', 'TEXT', 0, None, 0), (7, 'headers', 'TEXT', 0, None, 0), (8, 'body', 'TEXT', 0, None, 0)], 'test_http': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'command', 'TEXT', 0, None, 0), (6, 'path', 'TEXT', 0, None, 0), (7, 'headers', 'TEXT', 0, None, 0), (8, 'body', 'TEXT', 0, None, 0)], 'p0f': [(0, 'session', 'TEXT', 1, None, 1), (1, 'first_seen', 'TEXT', 0, None, 0), (2, 'last_seen', 'TEXT', 0, None, 0), (3, 'uptime', 'INTEGER', 0, None, 0), (4, 'last_nat', 'TEXT', 0, None, 0), (5, 'last_chg', 'TEXT', 0, None, 0), (6, 'distance', 'INTEGER', 0, None, 0), (7, 'bad_sw', 'INTEGER', 0, None, 0), (8, 'os_name', 'TEXT', 0, None, 0), (9, 'os_flavor', 'TEXT', 0, None, 0), (10, 'os_match_q', 'INTEGER', 0, None, 0), (11, 'http_name', 'TEXT', 0, None, 0), (12, 'http_flavor', 'TEXT', 0, None, 0), (13, 'total_conn', 'INTEGER', 0, None, 0), (14, 'link_type', 'TEXT', 0, None, 0), (15, 'language', 'TEXT', 0, None, 0)], 'ipInfo': [(0, 'ip', 'TEXT', 1, None, 1), (1, 'plugin_instance', 'TEXT', 1, None, 2), (2, 'timestamp', 'TEXT', 1, None, 0), (3, 'hostname', 'TEXT', 0, None, 0), (4, 'city', 'TEXT', 0, None, 0), (5, 'region', 'TEXT', 0, None, 0), (6, 'country', 'TEXT', 0, None, 0), (7, 'lat', 'REAL', 0, None, 0), (8, 'long', 'REAL', 0, None, 0), (9, 'org', 'TEXT', 0, None, 0), (10, 'postal', 'TEXT', 0, None, 0)], 'test_telnet': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'input_type', 'TEXT', 0, None, 0), (6, 'user_input', 'TEXT', 0, None, 0)]} @patch.object(Logger,'__new__') def test_get_schema_from_database(self,log): # will call the constructor because this calls this method db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() self.assertTrue(set(validator.tables) == set(self.tables_test)) self.assertEqual(validator.table_schema,self.table_schema_test) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_get_tables(self,log): db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() self.assertIsInstance(validator.get_tables(),list) self.assertTrue(set(validator.get_tables()) == set(self.tables_test)) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_get_schema(self,log): db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() self.assertIsInstance(validator.get_schema(),dict) self.assertEqual(validator.get_schema(),self.table_schema_test) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_value_len(self, log): good_dict = {'table1':{'col1':'val1','col2':'val2'}} bad_dict = {'table1':{'col1':'val1','col2':'val2'},'table2':{'col3':'val3','col4':'val4'}} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_value_len(good_dict)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_value_len(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_value_is_dict(self,log): good_dict = {'table1':{'col1':'val1','col2':'val2'}} bad_dict = ['im a list'] db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_value_is_dict(good_dict)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_value_is_dict(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_key_in_dict_string(self,log): good_dict = {'table1':{'col1':'val1','col2':'val2'}} bad_dict = {1:{'col1':'val1'}} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_key_in_dict_string(good_dict)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_key_in_dict_string(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_key_is_valid_table_name(self,log): good_dict = {'test_http':{'col1':'val1','col2':'val2'}} bad_dict = {'test_table':{'col1':'val1'}} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_key_is_valid_table_name(good_dict)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_key_is_valid_table_name(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_row_value_is_dict(self,log): good_dict = {'test_table':{'col1':'val1','col2':'val2'}} bad_dict = {'test_table':'i am not a dictionary'} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_row_value_is_dict(good_dict)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_row_value_is_dict(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_all_col_names_strings(self,log): good_dict = {'test_table':{'col1':'val1','col2':'val2'}} bad_dict = {'test_table':{1:'val1','string':'val2'}} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() self.assertTrue(validator.check_all_col_names_strings(good_dict)) self.assertFalse(validator.check_all_col_names_strings(bad_dict)) shutil.rmtree(os.getcwd() + self.test_db_dir) @patch.object(Logger,'__new__') def test_check_all_col_exist(self,log): good_dict = {'test_http':{'session': 'val1','eventDateTime': 'val2','peerAddress': 'val3', 'localAddress': 'val4','command': 'val5','path': 'val6', 'headers': 'val7','body': 'val8'}} bad_dict = {'test_http2':{'session': 'val1','eventDateTime': 'val2','peerAddress': 'val3', 'XYZ': 'val4','command': 'val5','path': 'val6', 'headers': 'val7','body': 'val8'}} missing_col = {'test_http':{'session': 'val1','eventDateTime': 'val2','peerAddress': 'val3', 'localAddress': 'val4', 'path': 'val6', 'headers': 'val7', 'body': 'val8'}} db = database.Database() db.create_default_database() validator = datavalidator.DataValidator() log.error = unittest.mock.Mock() log.error.reset_mock() self.assertTrue(validator.check_all_col_exist(good_dict)) self.assertFalse(db.log.error.called) self.assertTrue(validator.check_all_col_exist(missing_col)) self.assertFalse(db.log.error.called) self.assertFalse(validator.check_all_col_exist(bad_dict)) self.assertTrue(db.log.error.called) shutil.rmtree(os.getcwd() + self.test_db_dir)
def setUp(self): self.test_db_dir = '/tests/database/test_database' self.test_db_file = '/tests/database/test_database/honeyDB.sqlite' # test configuration files self.plugins_config_file = 'tests/database/test_config/plugins.cfg' self.plugins_config_diff_file = 'tests/database/test_config/plugins_diff.cfg' self.plugins_config_diff_table_file = 'tests/database/test_config/plugins_diff_table.cfg' self.global_config_file = 'tests/database/test_config/global.cfg' # create global config instance self.gci = GlobalConfig(self.plugins_config_file,self.global_config_file) self.gci.read_global_config() self.gci.read_plugin_config() self.tables_test = ['p0f', 'ipInfo', 'sessions', 'test_http', 'test_http2', 'test_telnet'] self.table_schema_test = {'sessions': [(0, 'session', 'TEXT', 0, None, 1), (1, 'table_name', 'TEXT', 1, None, 2)], 'test_http2': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'command', 'TEXT', 0, None, 0), (6, 'path', 'TEXT', 0, None, 0), (7, 'headers', 'TEXT', 0, None, 0), (8, 'body', 'TEXT', 0, None, 0)], 'test_http': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'command', 'TEXT', 0, None, 0), (6, 'path', 'TEXT', 0, None, 0), (7, 'headers', 'TEXT', 0, None, 0), (8, 'body', 'TEXT', 0, None, 0)], 'p0f': [(0, 'session', 'TEXT', 1, None, 1), (1, 'first_seen', 'TEXT', 0, None, 0), (2, 'last_seen', 'TEXT', 0, None, 0), (3, 'uptime', 'INTEGER', 0, None, 0), (4, 'last_nat', 'TEXT', 0, None, 0), (5, 'last_chg', 'TEXT', 0, None, 0), (6, 'distance', 'INTEGER', 0, None, 0), (7, 'bad_sw', 'INTEGER', 0, None, 0), (8, 'os_name', 'TEXT', 0, None, 0), (9, 'os_flavor', 'TEXT', 0, None, 0), (10, 'os_match_q', 'INTEGER', 0, None, 0), (11, 'http_name', 'TEXT', 0, None, 0), (12, 'http_flavor', 'TEXT', 0, None, 0), (13, 'total_conn', 'INTEGER', 0, None, 0), (14, 'link_type', 'TEXT', 0, None, 0), (15, 'language', 'TEXT', 0, None, 0)], 'ipInfo': [(0, 'ip', 'TEXT', 1, None, 1), (1, 'plugin_instance', 'TEXT', 1, None, 2), (2, 'timestamp', 'TEXT', 1, None, 0), (3, 'hostname', 'TEXT', 0, None, 0), (4, 'city', 'TEXT', 0, None, 0), (5, 'region', 'TEXT', 0, None, 0), (6, 'country', 'TEXT', 0, None, 0), (7, 'lat', 'REAL', 0, None, 0), (8, 'long', 'REAL', 0, None, 0), (9, 'org', 'TEXT', 0, None, 0), (10, 'postal', 'TEXT', 0, None, 0)], 'test_telnet': [(0, 'ID', 'INTEGER', 1, None, 1), (1, 'session', 'TEXT', 0, None, 0), (2, 'eventDateTime', 'TEXT', 0, None, 0), (3, 'peerAddress', 'TEXT', 0, None, 0), (4, 'localAddress', 'TEXT', 0, None, 0), (5, 'input_type', 'TEXT', 0, None, 0), (6, 'user_input', 'TEXT', 0, None, 0)]}
trace_command = "tracert -d " else: trace_command = "traceroute -n " # Spitball test routes via Python's traceroute for endpoint in endpoints: log.info(f" {trace_command}{endpoint}") # Register Signal Handler for Interrupt signal(SIGINT, interrupt_handler) main.add_command(amazon) main.add_command(test_notifications) main.add_command(show) main.add_command(find_endpoints) main.add_command(show_traceroutes) # Global scope stuff here if is_latest(): log.info(f"FairGame v{version}") elif version.is_prerelease: log.warning(f"FairGame PRE-RELEASE v{version}") else: log.warning( f"You are running FairGame v{version}, but the most recent version is v{get_latest_version()}. " f"Consider upgrading ") global_config = GlobalConfig() notification_handler = NotificationHandler()
class Database: def __init__(self): self.global_config = GlobalConfig() self.log = Logger().get('database.database.Database') def create_default_database(self): """ Calls methods needed to create the database. """ self.create_db_dir() self.create_db() # Execute scripts BEFORE updating schema run_db_scripts(self.global_config) self.update_schema() def create_db_dir(self): """ Creates the database directory if it doesn't already exist. """ # if database directory does not exist create it db_path = self.global_config['Database']['path'] (db_dir, db_name) = ntpath.split(db_path) if not os.path.isdir(db_dir): self.log.info("Database directory not found, " "creating database directory...") os.mkdir(db_dir) def create_db(self): """ Creates the database if it doesn't already exist. """ # if database file does not exist in the directory, create it (db_dir, db_name) = ntpath.split(self.global_config['Database']['path']) if not os.path.exists(self.global_config['Database']['path']): self.log.info("Database file not found, creating database file...") # this actually creates the database file connection = sqlite3.connect(self.global_config['Database']['path']) connection.close() def update_schema(self): """ Updates the database when columns have been added to, or removed from, the schema. """ # Create any new tables that have been added to the plugin # config schema. db_tables = DataValidator().get_tables() cfg_tables = get_config_table_list( self.global_config.get_ports(), self.global_config.get_plugin_dictionary()) table_diff = list(set(cfg_tables) - set(db_tables)) self.create_non_exist_tables(table_diff) # Populate the newly created tables with their column # definitions. DataValidator().update_tables_and_schema() self.update_table_structure() def create_non_exist_tables(self, table_diff): """ create tables that do not exist from the table difference between the current database and the configuration """ if len(table_diff) > 0: for table in table_diff: Table_Init.create_table(table, self.global_config) self.log.info('Updated database schema, table names now match configuration.') else: self.log.info('Database Schema and Configuration table names already match.') def create_dict_config_column_list(self): """ get a dictionary of tables and corresponding columns from the config """ config_column_lists = {} for port in self.global_config.get_ports(): value = self.global_config.get_plugin_dictionary().get(port) config_column_lists[value.get('table')] = value.get('tableColumns') return config_column_lists def create_dict_transformed_column_list(self, database_column_lists): """ returns only custom plugin defined columns from database schema i.e. ignores default columns """ transformed_db_column_list = {} for table in database_column_lists: col_list = database_column_lists[table] transformed_db_column_list[table] = [] # default column ids to ignore default_list = [] for default in default_columns: default_list.append(default[0]) for column in col_list: # ignores the default columns if column[1] in default_list: continue transformed_db_column_list[table].append([column[0],column[1],column[2]]) return transformed_db_column_list def update_table_structure(self): cfg_schema = self.create_dict_config_column_list() db_schema = DataValidator().get_schema() db_schema_sans_defaults = self.create_dict_transformed_column_list(db_schema) for table in cfg_schema: if not [(x[1], x[2]) for x in cfg_schema[table]] == \ [(x[1], x[2]) for x in db_schema_sans_defaults[table]]: Table_Init.change_table_structure( table, cfg_schema[table], db_schema[table], self.global_config)
def __init__(self): self.g_config = GlobalConfig() self.valid_port_numbers = self.g_config.get_ports() self.date_time_field = self.g_config.get_db_datetime_name() self.log = Logger().get('reportserver.manager.IpsManager.py')
def __init__(self): self.globalconfig = GlobalConfig() self.nodes = self.globalconfig.get_hosts() self.notifications = Notifications() self.application = Applications() self.logger = logging.getLogger(__name__)
class GlobalConfig_Test(unittest.TestCase): def setUp(self): self.gconfig = GlobalConfig(test_cfg_path, test_global_cfg_path, refresh=True) self.gconfig.read_plugin_config() self.gconfig.read_global_config() def test_getInstance(self): gconfig2 = GlobalConfig() self.assertEqual(str(self.gconfig),str(gconfig2),"these 2 objects should equal") gconfig3 = GlobalConfig() self.assertEqual(str(self.gconfig), str(gconfig3), "these 2 objects should equal") self.assertEqual(str(gconfig2), str(gconfig3), "these 2 objects should equal") def test_getPorts(self): ports = self.gconfig.get_ports() self.assertEqual(len(ports), 2, "expected 2 ports in test.cfg found: " + str(len(ports))) for port in ports: print("found: " + str(port)) def test_getReportServerConfig(self): host = self.gconfig.get_report_server_host() port = self.gconfig.get_report_server_port() self.assertEqual(host, "", "expected host to be ''") self.assertEqual(port, 8080, "expected port to be '8080' ") def test_getReportServerHost(self): self.assertEqual("", self.gconfig.get_report_server_host()) def test_getReportServerPort(self): self.assertEqual(8080, self.gconfig.get_report_server_port()) def test_refresh_instance(self): gconfig2 = GlobalConfig(test_cfg_path, test_global_cfg_path, refresh=True) self.assertNotEqual(str(self.gconfig), str(gconfig2), "these 2 objects should NOT equal when refresh set to True") def test_refresh_instance_same(self): gconfig2 = GlobalConfig() self.assertEqual(str(self.gconfig), str(gconfig2), "these 2 objects should equal when False is set for Refresh") gconfig2 = GlobalConfig() self.assertEqual(str(self.gconfig), str(gconfig2), "these 2 objects should equal with default of False") def test_get_date_time_name(self): self.assertEqual("eventDateTime", self.gconfig['Database']['datetime.name']) def test_get_db_peerAddress_nameself(self): self.assertEqual("peerAddress", self.gconfig['Database']['peerAddress.name']) def test_get_db_localAddress_name(self): self.assertEqual("localAddress", self.gconfig['Database']['localAddress.name'])
def __init__(self): self.log = Logger().get('reportserver.manager.WorldmapServiceManager.py') self.global_config = GlobalConfig() self.global_config.read_plugin_config() self.global_config.read_global_config()
class RestRequestHandler (BaseHTTPRequestHandler): def do_GET(self) : path_query_tuple = utilities.get_path_query_tokens(self.path) path_tokens = path_query_tuple[0] query_tokens = path_query_tuple[1] if self.path.startswith("/v1/analytics"): if len(path_tokens) >= 4: if str(path_tokens[3]) == "ports": PortsServiceHandler().process(self, path_tokens, query_tokens) elif str(path_tokens[3]) == "ipaddresses": IpsServiceHandler().process(self, path_tokens, query_tokens) elif str(path_tokens[3]) == "worldmap": WorldmapServiceHandler().process(self, path_tokens, query_tokens) elif str(path_tokens[3] == ""): self.showIndex() else: self.badRequest() else: self.showIndex() else: self.notFound() def get_full_url_path(self): self.g_config = GlobalConfig() self.host = self.g_config.get_report_server_host() self.port = self.g_config.get_report_server_port() full_path = 'http://%s:%s/v1/analytics' % (str(self.host), str(self.port)) return full_path def getIndexPayload(self): return {'links': ['rel: ports, href: ' + self.get_full_url_path() + '/ports', 'rel: ipaddresses, href:' + self.get_full_url_path() + '/ipaddresses']} def showIndex(self): # send response code: self.sendJsonResponse(self.getIndexPayload(), 200) def notFound(self): # send response code: self.sendJsonResponse(notFoundPayload,404) def badRequest(self, rqstPayload=badRequestPayload): # send response code: self.sendJsonResponse(rqstPayload,400) def sendJsonResponse(self, payload, responseCode): # Note: responseCode must be set before headers in python3!! # see this post: # http://stackoverflow.com/questions/23321887/python-3-http-server-sends-headers-as-output/35634827#35634827 json_result = json.dumps(payload) self.send_response(responseCode) #todo make this configurable for allow-origin self.send_header("Access-Control-Allow-Origin","http://localhost:8000") self.send_header('Content-Type', 'application/json') self.send_header('Content-Length', len(json_result)) self.end_headers() self.flush_headers() self.wfile.write(bytes(json_result, "utf-8")) self.wfile.flush() return def sendPngResponse(self, filepath, responseCode): # Note: responseCode must be set before headers in python3!! # see this post: # http://stackoverflow.com/questions/23321887/python-3-http-server-sends-headers-as-output/35634827#35634827 f=open(filepath, 'rb') self.send_response(responseCode) # todo make this configurable for allow-origin self.send_header("Access-Control-Allow-Origin", "http://localhost:8000") self.send_header('Content-Type', 'image/png') #self.send_header('Content-Length', len(json_result)) self.end_headers() self.flush_headers() self.wfile.write(f.read()) self.wfile.flush() f.close() return
def setUp(self): self.gconfig = GlobalConfig(test_cfg_path, test_global_cfg_path, refresh=True) self.gconfig.read_plugin_config() self.gconfig.read_global_config()
def get_full_url_path(self): self.g_config = GlobalConfig() self.host = self.g_config.get_report_server_host() self.port = self.g_config.get_report_server_port() full_path = 'http://%s:%s/v1/analytics' % (str(self.host), str(self.port)) return full_path