def simple_master_server(): master = RedisServer(unix_socket=REDIS_SOCKET_PATH) filter = Filter(filter_name='test') filter.configure(FTEST_CONF_TEMPLATE) filter.valgrind_start() try: filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error( "simple_master_server: Could not connect to test filter: {}". format(e)) return False sleep(1) with master.connect() as redis_connection: num_list_entries = redis_connection.llen(REDIS_LIST_NAME) if num_list_entries != 1: logging.error( "simple_master_server: wrong number of entries in the redis list {}: " + "expected 1 but got {}".format(REDIS_LIST_NAME, str(num_list_entries))) return False return True
def multi_thread_master(): master = RedisServer(unix_socket="/tmp/redis.socket") filter = Logs(redis_server=master, nb_threads=5) filter.configure(FLOGS_CONF_TEMPLATE) filter.start() thread_list = [] def thread_brute(filter, count_log): for count in range(0, count_log): try: filter.log(b'All work and no play makes Jake a dull boy.') except: return False return True for num in range(0, 5): thread_list.append(threading.Thread(target=thread_brute, args=(filter, 50))) for thread in thread_list: thread.start() for thread in thread_list: thread.join() sleep(1) number = master.get_number_of_connections() if number != 6: logging.error("master_timeout: wrong number active connections: expected 6 but got " + str(number)) return False return True
def single_log_to_all(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_ALL) if not filter.valgrind_start(): return False redis_server.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.log(b'There are only two things wrong with C++: The initial concept and the implementation. Bertrand Meyer\n') sleep(1) except Exception as e: logging.error("single_log_to_all: Unable to connect to filter: {}".format(e)) return False logs = filter.get_logs_from_file() if logs != ['There are only two things wrong with C++: The initial concept and the implementation. Bertrand Meyer\n']: logging.error("single_log_to_all: Log line not matching: {}".format(logs)) return False redis_log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if redis_log != 'There are only two things wrong with C++: The initial concept and the implementation. Bertrand Meyer\n': logging.error("single_log_to_all: Log line not matching: {}".format(log)) return False redis_channel_log = redis_server.channel_get_message() if redis_channel_log != 'There are only two things wrong with C++: The initial concept and the implementation. Bertrand Meyer\n': logging.error("single_log_to_all: Log line not matching: {}".format(log)) return False return True
def empty_log_to_all(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_ALL) if not filter.valgrind_start(): return False redis_server.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.log(b'') sleep(1) except Exception as e: logging.error("empty_log_to_all: Unable to connect to filter: {}".format(e)) return False logs = filter.get_logs_from_file() if logs != []: logging.error("empty_log_to_all: Log line not matching: {}".format(logs)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if log is not None: logging.error("empty_log_to_all: Log line not matching: {}".format(log)) return False channel_log = redis_server.channel_get_message() if channel_log is not None: logging.error("empty_log_to_all: channel log line not matching: {}".format(channel_log)) return False return True
def master_replica_discovery_rate_limiting(): master = RedisServer(address="127.0.0.1", port=1234) replica = RedisServer(unix_socket=REDIS_SOCKET_PATH, master=master) filter = Filter(filter_name='test', nb_threads=5) filter.configure(FTEST_CONF_TEMPLATE) filter.start() try: # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error( "master_replica_discovery_rate_limiting: Could not connect to test filter: {}" .format(function_name, e)) return False # master shuts down master.stop() #brutal def thread_brute_time(filter, time_end): end = time() + time_end while time() < end: # accelerate throughput by avoiding constant calls to time() for i in range(0, 9): try: filter.send_single(REDIS_LIST_TRIGGER) except: return False return True thread_list = [] for num in range(0, 5): thread_list.append( threading.Thread(target=thread_brute_time, args=(filter, 9))) # ought to crash if command fails initial_connections_num = replica.connect().info( )['total_connections_received'] for thread in thread_list: thread.start() for thread in thread_list: thread.join() # new generated connections generated, minus the one generated by the call to get it new_connections = replica.connect().info( )['total_connections_received'] - initial_connections_num - 1 if new_connections > 10: logging.error( "master_replica_discovery_rate_limiting: Wrong number of new connection attempts, " + "was supposed to have 10 new at most, but got ".format( new_connections)) return False return True
def multiple_log_to_redis(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_REDIS) if not filter.valgrind_start(): return False try: filter.log(b'Computers are good at following instructions, but not at reading your mind. Donald Knuth\n') filter.log(b'If we wish to count lines of code, we should not regard them as lines produced but as lines spent. Edsger Dijkstra\n') sleep(1) except Exception as e: logging.error("multiple_log_to_redis: Unable to connect to filter: {}".format(e)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if log != 'Computers are good at following instructions, but not at reading your mind. Donald Knuth\n': logging.error("multiple_log_to_redis: Log line not matching: {}".format(log)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if log != 'If we wish to count lines of code, we should not regard them as lines produced but as lines spent. Edsger Dijkstra\n': logging.error("multiple_log_to_redis: Log line not matching: {}".format(log)) return False return True
def multiple_log_to_redis_channel(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_REDIS_CHANNEL) if not filter.valgrind_start(): return False redis_server.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.log(b'Computers are good at following instructions, but not at reading your mind. Donald Knuth\n') filter.log(b'If we wish to count lines of code, we should not regard them as lines produced but as lines spent. Edsger Dijkstra\n') sleep(1) except Exception as e: logging.error("multiple_log_to_redis_channel: Unable to connect to filter: {}".format(e)) return False log = redis_server.channel_get_message() if log != 'Computers are good at following instructions, but not at reading your mind. Donald Knuth\n': logging.error("multiple_log_to_redis_channel: Log line not matching: {}".format(log)) return False log = redis_server.channel_get_message() if log != 'If we wish to count lines of code, we should not regard them as lines produced but as lines spent. Edsger Dijkstra\n': logging.error("multiple_log_to_redis_channel: Log line not matching: {}".format(log)) return False log = redis_server.channel_get_message() if log is not None: logging.error("multiple_log_to_redis_channel: Log line not matching: waited '' but got '{}'".format(log)) return False return True
def master_slave_master_off(): master = RedisServer(address="127.0.0.1", port=1234) slave = RedisServer(unix_socket="/tmp/redis.socket", master=master) filter = Logs(redis_server=slave) filter.configure(FLOGS_CONF_TEMPLATE) master.stop() filter.start() try: filter.log(b'It s dangerous out there. Take this sword.') except Exception: logging.error("master_slave_master_off: Could not connect to logs filter") return False if not filter.check_run(): logging.error("master_slave_master_off: filter seems to have crashed when master Redis got offline") return False sleep(1) number = slave.get_number_of_connections() if number != 2: logging.error("master_slave_master_off: filter is not connected to slave") return False return True
def __init__(self, log_file=None, redis_server=None, nb_threads=1): super().__init__(filter_name="logs", nb_thread=nb_threads) self.log_file = log_file if log_file else LOG_FILE self.redis = redis_server if redis_server else RedisServer(unix_socket=REDIS_SOCKET) self.pubsub = None try: os.remove(self.log_file) except: pass
def master_slave(): master = RedisServer(address="127.0.0.1", port=1234) slave = RedisServer(unix_socket="/tmp/redis.socket", master=master) filter = Logs(redis_server=slave) filter.configure(FLOGS_CONF_TEMPLATE) filter.start() try: filter.log(b'It s dangerous out there. Take this sword.') except Exception: logging.error("master_slave: Could not connect to logs filter") return False sleep(1) numberMaster = master.get_number_of_connections() numberSlave = slave.get_number_of_connections() # master : python client, filter connection, slave ping # slave : python client, filter connection, master ping if numberMaster != 3 and numberSlave != 2: logging.error("master_slave: expected 2 master and 1 slave connections, but got " + str(numberMaster) + " and " + str(numberSlave)) return False return True
def master_replica_failover(function_name, healthcheck): master = RedisServer(address="127.0.0.1", port=1234) replica = RedisServer(unix_socket=REDIS_SOCKET_PATH, master=master) filter = Filter(filter_name='test') filter.configure(FTEST_CONF_TEMPLATE) filter.valgrind_start() try: # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error("{}: Could not connect to test filter: {}".format( function_name, e)) return False # master shuts down master.stop() #replica becomes master with replica.connect() as replica_connection: replica_connection.slaveof() sleep(1) if healthcheck: sleep(8) try: # success return_code = filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error("{}: Could not connect to test filter: {}".format( function_name, e)) return False if return_code != 0: logging.error( "{}: Filter didn't return correct code, " + "waited for 0 but got {}".format(function_name, return_code)) return False with replica.connect() as new_master_connection: num_entries = new_master_connection.llen(REDIS_LIST_NAME) if num_entries != 2: logging.error("{}: Wrong number of entries in {}, " + "expected 2 but got {}".format( function_name, REDIS_LIST_NAME, num_entries)) return False return True
def simple_master_server(): master = RedisServer(unix_socket="/tmp/redis.socket") filter = Logs(redis_server=master) filter.configure(FLOGS_CONF_TEMPLATE) filter.start() try: filter.log(b'The cake is a lie') except Exception: logging.error("simple_master_server: Could not connect to logs filter") return False sleep(1) number = master.get_number_of_connections() if number != 2: logging.error("simple_master_server: wrong number active connections: expected 2 but got " + str(number)) return False return True
def multi_thread_master(): master = RedisServer(unix_socket=REDIS_SOCKET_PATH) filter = Filter(filter_name='test', nb_threads=5) filter.configure(FTEST_CONF_TEMPLATE) filter.valgrind_start() thread_list = [] def thread_brute(filter, count_log): for count in range(0, count_log): try: filter.send_single(REDIS_LIST_TRIGGER) except: return False return True for num in range(0, 5): thread_list.append( threading.Thread(target=thread_brute, args=(filter, 500))) for thread in thread_list: thread.start() for thread in thread_list: thread.join() sleep(1) number = master.get_number_of_connections() # 5 threads if number != 5: logging.error( "multi_thread_master: wrong number of active connections: expected 5 but got " + str(number)) return False return True
def single_log_to_redis_channel(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_REDIS_CHANNEL) if not filter.valgrind_start(): return False redis_server.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.log(b'Surveillance is the business model of the Internet. Bruce Schneier\n') sleep(1) except Exception as e: logging.error("single_log_to_redis_channel: Unable to connect to filter: {}".format(e)) return False log = redis_server.channel_get_message() if log != 'Surveillance is the business model of the Internet. Bruce Schneier\n': logging.error("single_log_to_redis_channel: Log line not matching: {}".format(log)) return False return True
def master_timeout_restart(): master = RedisServer(unix_socket="/tmp/redis.socket") filter = Logs(redis_server=master) filter.configure(FLOGS_CONF_TEMPLATE) filter.start() try: filter.log(b'The cake is a lie') except Exception: logging.error("master_timeout: Could not connect to logs filter") return False # Wait for Janitor to consider connection has timed out and disconnect sleep(25) number = master.get_number_of_connections() if number != 1: logging.error("master_timeout: wrong number active connections: expected 1 but got " + str(number)) return False try: filter.log(b'The cake is a lie') except Exception: logging.error("master_timeout: Could not connect to logs filter") return False sleep(1) number = master.get_number_of_connections() if number != 2: logging.error("master_timeout: wrong number active connections: expected 2 but got " + str(number)) return False return True
def single_log_to_redis(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_REDIS) if not filter.valgrind_start(): return False try: filter.log(b'If you think your users are idiots, only idiots will use it. Linus Torvald\n') sleep(1) except Exception as e: logging.error("single_log_to_redis: Unable to connect to filter: {}".format(e)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if log != 'If you think your users are idiots, only idiots will use it. Linus Torvald\n': logging.error("single_log_to_redis: Log line not matching: {}".format(log)) return False return True
def empty_log_to_redis(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_REDIS) if not filter.valgrind_start(): return False try: filter.log(b'') sleep(1) except Exception as e: logging.error("empty_log_to_redis: Unable to connect to filter: {}".format(e)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) if log is not None: logging.error("empty_log_to_redis: Log line not matching: {}".format(log)) return False return True
def multiple_log_to_all(): redis_server = RedisServer(unix_socket=REDIS_SOCKET) filter = Logs() filter.configure(FLOGS_CONFIG_ALL) if not filter.valgrind_start(): return False redis_server.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.log(b'UNIX is simple. It just takes a genius to understand its simplicity. Denis Ritchie\n') filter.log(b'A hacker is someone who enjoys playful cleverness, not necessarily with computers. Richard Stallman\n') sleep(1) except Exception as e: logging.error("multiple_log_to_all: Unable to connect to filter: {}".format(e)) return False logs = filter.get_logs_from_file() if logs != [ 'UNIX is simple. It just takes a genius to understand its simplicity. Denis Ritchie\n', 'A hacker is someone who enjoys playful cleverness, not necessarily with computers. Richard Stallman\n' ]: logging.error("multiple_log_to_all: Log line not matching: {}".format(logs)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) channel_log = redis_server.channel_get_message() if log != 'UNIX is simple. It just takes a genius to understand its simplicity. Denis Ritchie\n': logging.error("multiple_log_to_all: Log line not matching: {}".format(log)) return False if log != channel_log: logging.error("multiple_log_to_all: channel log line not matching: {}".format(channel_log)) return False log = Logs.get_log_from_redis(redis_server, REDIS_LIST_NAME) channel_log = redis_server.channel_get_message() if log != 'A hacker is someone who enjoys playful cleverness, not necessarily with computers. Richard Stallman\n': logging.error("multiple_log_to_all: Log line not matching: {}".format(log)) return False if log != channel_log: logging.error("multiple_log_to_all: channel log line not matching: {}".format(channel_log)) return False return True
def master_replica(): master = RedisServer(address="127.0.0.1", port=1234) replica = RedisServer(address="127.0.0.1", port=1235, unix_socket=REDIS_SOCKET_PATH, master=master) filter = Filter(filter_name='test') filter.configure(FTEST_CONF_TEMPLATE) filter.valgrind_start() master.channel_subscribe(REDIS_CHANNEL_NAME) try: filter.send_single(REDIS_CHANNEL_TRIGGER) except Exception as e: logging.error( "master_replica: Could not connect to test filter: {}".format(e)) return False sleep(1) message = master.channel_get_message() if message is '': logging.error( "master_replica: expected to get a message in channel {} " + "but got nothing".format(REDIS_CHANNEL_NAME)) return False if message != REDIS_CHANNEL_TRIGGER: logging.error( "master_replica: expected to get a message in channel {} saying '{}' " + "but got '{}' instead".format(REDIS_CHANNEL_NAME, REDIS_CHANNEL_TRIGGER, message)) return False return True
def __init__(self): super().__init__(filter_name="buffer", cache_size=1) self.redis = RedisServer(unix_socket=REDIS_SOCKET) self.test_data = None
def redis_reports(): ret = False part = 1 connections = [{ "name": "ip", "conf": "\"ip\": \"127.0.0.1\", \"port\": 7890", "partial_result": "" }, { "name": "ip_wrong_port", "conf": "\"ip\": \"127.0.0.1\", \"port\": 7891", "partial_result": "nodata" }, { "name": "ip_wrong_ip", "conf": "\"ip\": \"127.0.0.2\", \"port\": 7890", "partial_result": "nodata" }, { "name": "unix", "conf": "\"unix_path\": \"{}\"".format(DEFAULT_REDIS_SOCKET), "partial_result": "" }, { "name": "unix_wrong", "conf": "\"unix_path\": \"wrong_path\"", "partial_result": "nodata" }] report_methods = [ { "name": "list", "conf": "\"list\": \"{}\"".format(DEFAULT_REDIS_LIST), "partial_result": STAT_LOG_MATCH }, { "name": "channel", "conf": "\"channel\": \"{}\"".format(DEFAULT_REDIS_CHANNEL), "partial_result": STAT_LOG_MATCH }, ] for method in report_methods: for connection in connections: print("[{}/10]".format(part), end='', flush=True) redis_conf = "\"redis\": {{{connection}, {method}}},".format( connection=connection['conf'], method=method['conf']) expected_result = method['partial_result'] if connection[ 'partial_result'] == "" else connection['partial_result'] darwin_configure( CONF_TEMPLATE.substitute(log_path=DEFAULT_FILTER_PATH, conf_path=PATH_CONF_FLOGS, conf_redis=redis_conf, conf_file="", proc_stats="")) darwin_configure(CONF_FLOGS, path=PATH_CONF_FLOGS) redis_server = RedisServer(unix_socket=DEFAULT_REDIS_SOCKET, address="127.0.0.1", port=7890) redis_probe = redis_server.connect() process = darwin_start() if method['name'] == "channel": redis_pubsub = redis_probe.pubsub( ignore_subscribe_messages=True) redis_pubsub.subscribe(DEFAULT_REDIS_CHANNEL) # Remove subscribe notification message sleep(0.1) redis_pubsub.get_message() sleep(1) message = redis_pubsub.get_message() try: message = str(message['data']) except Exception: message = "nodata" elif method['name'] == "list": sleep(1) message = redis_probe.lpop(DEFAULT_REDIS_LIST) if not message: message = "nodata" else: message = str(message) ret = True if expected_result in message else False if not ret: logging.error( "redis_reports(): while trying '{}' with '{}', didn't find '{}' in '{}'" .format(method['conf'], connection['conf'], expected_result, message)) del redis_server del message darwin_stop(process) darwin_remove_configuration() darwin_remove_configuration(path=PATH_CONF_FLOGS) # Don't judge me... part += 1 print("\x08\x08\x08\x08\x08\x08", end='', flush=True) if not ret: break if not ret: break # Dirty much... But pretty printing! if part == 11: print("\x08", end='', flush=True) print(" ", end='', flush=True) print("\x08\x08\x08\x08\x08\x08\x08", end='', flush=True) return ret
def file_and_redis_simple_report(): ret = False file_conf = """"file": {{"filepath": "{}"}},""".format(DEFAULT_STATS_FILE) redis_conf = """"redis": {{"ip": "127.0.0.1", "port": 7890, "channel": "{}", "list": "{}"}},""".format( DEFAULT_REDIS_CHANNEL, DEFAULT_REDIS_LIST) darwin_configure( CONF_TEMPLATE.substitute(log_path=DEFAULT_FILTER_PATH, conf_path=PATH_CONF_FLOGS, conf_redis=redis_conf, conf_file=file_conf, proc_stats="")) darwin_configure(CONF_FLOGS, path=PATH_CONF_FLOGS) redis_server = RedisServer(address="127.0.0.1", port=7890) redis_probe = redis_server.connect() redis_pubsub = redis_probe.pubsub(ignore_subscribe_messages=True) redis_pubsub.subscribe(DEFAULT_REDIS_CHANNEL) # Remove subscribe notification message sleep(0.1) redis_pubsub.get_message() process = darwin_start() try: with open(DEFAULT_STATS_FILE, "r") as statsfile: sleep(1) channel_message = redis_pubsub.get_message().get('data').decode( "utf-8") list_message = redis_probe.lpop(DEFAULT_REDIS_LIST).decode("utf-8") file_message = statsfile.readline() # File message has an additional '\n' if channel_message in file_message and list_message in file_message: if STAT_LOG_MATCH in file_message: ret = True else: logging.error( "file_and_redis_simple_report(): expected \"{}\", but got \"{}\"" .format(STAT_LOG_MATCH, file_message)) else: logging.error( "file_and_redis_simple_report(): messages are not identical" ) logging.error("file_and_redis_simple_report(): {}".format( channel_message)) logging.error( "file_and_redis_simple_report(): {}".format(list_message)) logging.error( "file_and_redis_simple_report(): {}".format(file_message)) except Exception as e: logging.error( "file_and_redis_simple_report(): could not open file -> {}".format( e)) try: os.remove(DEFAULT_STATS_FILE) except Exception: pass darwin_stop(process) darwin_remove_configuration() darwin_remove_configuration(path=PATH_CONF_FLOGS) return ret
def master_replica_master_temp_fail(): master = RedisServer(address="127.0.0.1", port=1234) replica = RedisServer(address="127.0.0.1", unix_socket=REDIS_SOCKET_PATH, master=master) filter = Filter(filter_name='test') filter.configure(FTEST_CONF_TEMPLATE) filter.valgrind_start() try: # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error( "master_replica_master_temp_fail: Could not connect to test filter: {}" .format(e)) return False master.stop() sleep(1) try: # failure filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error( "master_replica_master_temp_fail: Could not connect to test filter: {}" .format(e)) return False sleep(1) if not filter.check_run(): logging.error( "master_replica_master_temp_fail: filter seems to have crashed when master Redis got offline" ) return False master.start() # rate limiting will prevent to reconnect immediately after failure, so there should have a wait # (rate limiting is not tested here) sleep(8) try: # success filter.send_single(REDIS_LIST_TRIGGER) except Exception as e: logging.error( "master_replica_master_temp_fail: Could not connect to test filter: {}" .format(e)) return False with master.connect() as master_connection: num_list_entries = master_connection.llen(REDIS_LIST_NAME) if num_list_entries != 1: logging.error( "master_replica_master_temp_fail: wrong number of entries in the redis list {}: " + "expected 1 but got {}".format(REDIS_LIST_NAME, num_list_entries)) return False return True
def __init__(self): super().__init__(filter_name="tanomaly", cache_size=1) self.redis = RedisServer(unix_socket=REDIS_SOCKET) self.test_data = None self.internal_redis = self.filter_name + "_anomalyFilter_internal"
def __init__(self): super().__init__(filter_name='test') self.redis = RedisServer(unix_socket=REDIS_SOCKET)
def __init__(self, ): super().__init__(filter_name="connection") self.init_data_path = DEFAULT_INIT_DATA_PATH self.redis = RedisServer(unix_socket=DEFAULT_REDIS_SOCKET)