def test_mongodb_read_capped_db(database): """ Test read mongodb capped collection """ # Load DB mongodb = MongoDB(URI, "test_mongodb", "test_mongodb2") # Check if we can read one time mongodb.connect() mongodb_iter = mongodb.iter(HWPCModel(), False) report = None for _ in range(mongodb.collection.count_documents({})): report = next(mongodb_iter) assert report is not None # Check if there is nothing after with pytest.raises(StopIteration) as pytest_wrapped: next(mongodb_iter) assert pytest_wrapped.type == StopIteration # Add data in the collection for _ in range(1): mongodb.save(report, HWPCModel()) # Check if there is nothing after with pytest.raises(StopIteration) as pytest_wrapped: next(mongodb_iter) assert pytest_wrapped.type == StopIteration
def run(self): # Setup signal handler def term_handler(_, __): puller.send_kill() dispatcher.send_kill() pusher.send_kill() exit(0) signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) stream_mode = True supervisor = BackendSupervisor(stream_mode) # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result', HWPCModel()) pusher = PusherActor("pusher_mongodb", PowerReport, output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: RAPLFormulaActor( name, pusher, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep', HWPCModel(), stream_mode=stream_mode) report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) time.sleep(1) os.kill(dispatcher.pid, signal.SIGKILL) supervisor.join()
def test_puller_start_handler(self): """ Test the StartHandler of PullerActor """ # Define PullerState fake_database = get_fake_db() fake_socket_interface = get_fake_socket_interface() fake_filter = get_fake_filter() puller_state = PullerState(Mock(), fake_database, fake_filter, HWPCModel(), False, 100) puller_state.actor.receive_control = Mock(return_value=None) assert puller_state.initialized is False # Define StartHandler start_handler = PullerStartHandler(puller_state, 0.1) # Test Random message when state is not initialized to_send = [OKMessage(), ErrorMessage("Error"), create_report_root({})] for msg in to_send: start_handler.handle(msg) assert fake_database.method_calls == [] assert fake_socket_interface.method_calls == [] assert fake_filter.method_calls == [] assert puller_state.initialized is False # Try to initialize the state start_handler.handle(StartMessage()) assert puller_state.initialized is True
def test_csvdb_all_reports(clean_csv_files): """ Test create/save/read all kind of reports """ all_reports = [(PowerModel(), gen_power_report), (HWPCModel(), gen_hwpc_report)] for model, generator in all_reports: # Load DB csvdb = CsvDB(current_path=PATH_TO_SAVE) csvdb.connect() # Create report report = generator() # Save report csvdb.save(report, model) # Read report for r, d, f in os.walk(PATH_TO_SAVE + report.sensor + "-" + report.target + "/"): for file in f: if '.csv' in file: csvdb.add_file(os.path.join(r, file)) csvdb_iter = csvdb.iter(model, False) read_report = next(csvdb_iter) # Compare assert read_report == report
def test_ReportHandler_message_saving_order(): """ Handle 3 HWPCReport with a pusher.ReportHandler The maximum size of the handler buffer is 2 This 3 reports are not sent in their chronological order First report : timestamp = 0 Second report : timestamp = 2 Third report : timestamp = 1 When the handler save the received reports test if the reports are saved in their chronological order """ fake_database = get_fake_db() fake_database.save_many = Mock() pusher_state = PusherState(Mock(), fake_database, HWPCModel()) report_handler = ReportHandler(pusher_state, delay=10000, max_size=2) report0 = HWPCReport(datetime.fromtimestamp(0), None, None, None) report1 = HWPCReport(datetime.fromtimestamp(2), None, None, None) report2 = HWPCReport(datetime.fromtimestamp(1), None, None, None) report_handler.handle(report0) report_handler.handle(report1) report_handler.handle(report2) buffer = fake_database.save_many.call_args[0][0] assert len(buffer) == 3 assert buffer[0].timestamp < buffer[1].timestamp assert buffer[1].timestamp < buffer[2].timestamp
def test_with_two_filter(self, database): """ Test filter with two filter - 2 first report return first dispatcher - 2 next report return first and second dispatcher - 2 next report return None """ mongodb = MongoDB(URI, "test_filter", "test_filter1") mongodb.connect() hwpc_filter = Filter() hwpc_filter.filter(lambda msg: True if "sensor" in msg.sensor else False, 1) hwpc_filter.filter(lambda msg: True if "test1" in msg.sensor else False, 2) hwpc_filter.filter(lambda msg: True if msg.sensor == "sensor_test2" else False, 3) mongodb_it = mongodb.iter(HWPCModel(), False) for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1, 2] for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1, 3] for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1]
def test_csvdb_bad_filepath(self): """ Test with bad filepath """ with pytest.raises(CsvBadFilePathError) as pytest_wrapped: CsvDB(HWPCModel(), ["/tmp/unknowfile"]).connect() assert pytest_wrapped.type == CsvBadFilePathError
def __init__(self, component_group_name): Generator.__init__(self, component_group_name) self.model_factory = { 'HWPCReport': HWPCModel(), 'PowerReport': PowerModel(), 'FormulaReport': FormulaModel(), 'ControlReport': ControlModel(), } self.db_factory = { 'mongodb': lambda db_config: MongoDB(db_config['uri'], db_config['db'], db_config['collection']), 'socket': lambda db_config: SocketDB(db_config['port']), 'csv': lambda db_config: CsvDB( current_path=os.getcwd() if 'directory' not in db_config else db_config['directory'], files=[] if 'files' not in db_config else db_config['files']), 'influxdb': lambda db_config: InfluxDB(db_config['uri'], db_config['port'], db_config['db']), 'opentsdb': lambda db_config: OpenTSDB(db_config['uri'], db_config['port'], db_config['metric_name']), }
def test_pusher_start_handler(self): """ Test the StartHandler of PusherActor """ # Define PusherState fake_database = get_fake_db() fake_socket_interface = get_fake_socket_interface() pusher_state = PusherState(Mock(), fake_database, HWPCModel()) pusher_state.actor.socket_interface = fake_socket_interface assert pusher_state.initialized is False # Define StartHandler start_handler = PusherStartHandler(pusher_state) # Test Random message when state is not initialized to_send = [OKMessage(), ErrorMessage("Error"), create_report_root({})] for msg in to_send: start_handler.handle(msg) assert fake_database.method_calls == [] assert fake_socket_interface.method_calls == [] assert pusher_state.initialized is False # Try to initialize the state start_handler.handle(StartMessage()) assert pusher_state.initialized is True
def test_mongodb_all_reports(database): """ Test create/save/read all kind of reports """ all_reports = [(HWPCModel(), gen_hwpc_report), (PowerModel(), gen_power_report)] for model, generator in all_reports: # Load DB mongodb = MongoDB(URI, "test_reports" + model.get_type().__name__, "test_reports" + model.get_type().__name__) mongodb.connect() # Create report report = generator() # Save report mongodb.save(report, model) # Read report mongodb_iter = mongodb.iter(model, False) read_report = next(mongodb_iter) # Compare assert read_report == report
def test_mongodb_bad_port(database): """ Test if the database doesn't exist (hostname/port error) """ with pytest.raises(MongoBadDBError) as pytest_wrapped: MongoDB("mongodb://localhost:1", "error", "error", HWPCModel()).connect() assert pytest_wrapped.type == MongoBadDBError
def mongodb_database(uri, database_name, collection_name, stream_mode): """ Return MongoDB database """ database = MongoDB(uri, database_name, collection_name, HWPCModel(), stream_mode=stream_mode) return database
def test_csvdb_bad_common(self): """ Test when file miss some common column """ csv_files = BAD_COMMON while csv_files: with pytest.raises(CsvBadCommonKeysError) as pytest_wrapped: CsvDB(HWPCModel(), csv_files).connect() assert pytest_wrapped.type == CsvBadCommonKeysError csv_files = csv_files[1:]
async def test_read_one_json_object_received_from_the_socket(socket_db, unused_tcp_port): json_reports = extract_json_report(1) client = ClientThread(json_reports, unused_tcp_port) client.start() iterator = socket_db.iter(HWPCModel(), False) report = await iterator.__anext__() assert_report_equals(report, json_reports[0])
def test_crash_dispatcher(database, supervisor): # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result', HWPCModel()) pusher = PusherActor("pusher_mongodb", PowerReport, output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: RAPLFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep', HWPCModel()) report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) supervisor.join() check_db()
def test_mongodb_save_basic_db(database): """ Test save mongodb collection """ # Load DB mongodb = MongoDB(URI, "test_mongodb", "test_mongodb3") mongodb.connect() # Check if save work basic_count = mongodb.collection.count_documents({}) for _ in range(2): mongodb.save(gen_hwpc_report(), HWPCModel()) assert mongodb.collection.count_documents({}) == basic_count + 2
def puller(request, database, filt, stream_mode): """ Setup and Teardown for managing a PullerActor setup: create a PullerActor and start its process teardown: terminate the PullerActor process """ puller_actor = PullerActor("test_puller_" + str(request.node.name), database, filt, HWPCModel(), stream_mode=stream_mode, level_logger=LOG_LEVEL) yield puller_actor
def test_csvdb_second_primary_missing(self, csvdb): """ Create one full HWPCReport (the first), then return None """ csvdb.add_files(SECOND_PRIMARY_MISSING) csvdb.connect() group_name = [path.split('/')[-1][:-4] for path in SECOND_PRIMARY_MISSING] csvdb_iter = csvdb.iter(HWPCModel(), False) report = next(csvdb_iter) for group in group_name: assert group in report.groups assert report.timestamp == timestamp_to_datetime(1539260664189) with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def test_csvdb_two_reports(self, csvdb): """ Create two full HWPCReport, then return None """ csvdb.add_files(BASIC_FILES) csvdb.connect() group_name = [path.split('/')[-1][:-4] for path in BASIC_FILES] csvdb_iter = csvdb.iter(HWPCModel(), False) for _ in range(2): report = next(csvdb_iter) for group in group_name: assert group in report.groups with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor): """ run the same test but set a delay of 1s after sendig the 5th first messages """ # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL, max_size=0) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_socket = SocketDB(unused_tcp_port) report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_socket", input_socket, report_filter, HWPCModel(), level_logger=LOG_LEVEL, stream_mode=True) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) time.sleep(1) json_reports = extract_json_report(10) client = ClientThreadDelay(json_reports, unused_tcp_port) client.start() time.sleep(2) check_db()
def test_csvdb_second_primary_missing(self): """ Create one full HWPCReport (the first), then return None """ csvdb = CsvDB(HWPCModel(), SECOND_PRIMARY_MISSING) csvdb.connect() group_name = [path.split('/')[-1] for path in SECOND_PRIMARY_MISSING] csvdb_iter = iter(csvdb) report = next(csvdb_iter) for key in KEYS_COMMON: assert key in report for group in group_name: assert group in report['groups'] assert report['timestamp'] == "1539260664189" with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def test_mongodb_read_basic_db(database): """ Test read mongodb collection """ # Load DB mongodb = MongoDB(URI, "test_mongodb", "test_mongodb1") # Check if we can reload after reading mongodb.connect() for _ in range(2): mongodb_iter = mongodb.iter(HWPCModel(), False) for _ in range(10): assert next(mongodb_iter) is not None with pytest.raises(StopIteration) as pytest_wrapped: next(mongodb_iter) assert pytest_wrapped.type == StopIteration
def test_csvdb_second_rapl_missing(self, csvdb): """ Create two reports, one is full, second without rapl, then return None """ csvdb.add_files(SECOND_RAPL_MISSING) csvdb.connect() group_name = [path.split('/')[-1][:-4] for path in SECOND_RAPL_MISSING] csvdb_iter = csvdb.iter(HWPCModel(), False) for i in range(2): report = next(csvdb_iter) for group in group_name: if i == 1 and "rapl" in group: assert group not in report.groups else: assert group in report.groups with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def puller(request, database, stream_mode): """ Setup and Teardown for managing a PullerActor setup: create a PullerActor and start its process teardown: terminate the PullerActor process """ dispatcher = DispatcherActor('dispatcher__', Mock(), Mock()) filt = Filter() filt.filter(lambda msg: True, dispatcher) puller_actor = PullerActor("test_puller_" + str(request.node.name), database, filt, HWPCModel(), stream_mode=stream_mode, level_logger=LOG_LEVEL) return puller_actor
def test_csvdb_two_reports(self): """ Create two full HWPCReport, then return None """ csvdb = CsvDB(HWPCModel(), BASIC_FILES) csvdb.connect() group_name = [path.split('/')[-1] for path in BASIC_FILES] csvdb_iter = iter(csvdb) for _ in range(2): report = next(csvdb_iter) for key in KEYS_COMMON: assert key in report for group in group_name: assert group in report['groups'] with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def test_csvdb_second_rapl_missing(self): """ Create two reports, one is full, second without rapl, then return None """ csvdb = CsvDB(HWPCModel(), SECOND_RAPL_MISSING) csvdb.connect() group_name = [path.split('/')[-1] for path in SECOND_RAPL_MISSING] csvdb_iter = iter(csvdb) for i in range(2): report = next(csvdb_iter) for key in KEYS_COMMON: assert key in report for group in group_name: if i == 1 and "rapl" in group: assert group not in report['groups'] else: assert group in report['groups'] with pytest.raises(StopIteration) as pytest_wrapped: next(csvdb_iter) assert pytest_wrapped.type == StopIteration
def test_run(mongo_database, influx_database, supervisor): # Pusher output_db = InfluxDB(INFLUX_URI, INFLUX_PORT, INFLUX_DBNAME) pusher = PusherActor("pusher_influx", PowerModel(), output_db, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) supervisor.join() check_db(influx_database)
def test_run(database, supervisor): # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) t = time.time() number_of_output_reports = 0 for i in range(3): time.sleep(0.2) current = get_number_of_output_reports() assert current > number_of_output_reports number_of_output_reports = current time.sleep(0.1) supervisor.join()
def __init__(self, component_group_name): Generator.__init__(self, component_group_name) self.model_factory = { 'HWPCReport': HWPCModel(), 'PowerReport': PowerModel(), 'FormulaReport': FormulaModel(), 'ControlReport': ControlModel(), } self.db_factory = { 'mongodb': lambda db_config, _: MongoDB(db_config['uri'], db_config['db'], db_config['collection']), 'socket': lambda db_config, _: SocketDB(db_config['port']), 'csv': lambda db_config, _: CsvDB( current_path=os.getcwd() if 'directory' not in db_config else db_config['directory'], files=[] if 'files' not in db_config else db_config['files']), 'influxdb': lambda db_config, _: InfluxDB(db_config['uri'], db_config['port'], db_config['db']), 'opentsdb': lambda db_config, _: OpenTSDB(db_config['uri'], db_config['port'], db_config['metric_name']), 'prom': lambda db_config, _: PrometheusDB( db_config['port'], db_config['addr'], db_config['metric_name'], db_config['metric_description'], self.model_factory[db_config[ 'model']], db_config['aggregation_period']), 'tcp': lambda db_config, group_name: IOTcpDB(db_config['uri'], db_config['port'], input=(group_name == "input") ) }
def test_run(database, supervisor): """ Test the architecture when it has to deal with a lot of data, pulled with a hight frequency. Input database contain a lot of data, that will overflow the backend. Backend have to handle all of this data and continue to compute data and writing them in real time. Architecture : - 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off) - 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET) - 1 Dummy Formula - 1 pusher (connected to MongoDB1 [collection test_result] MongoDB1 content : - 3000 HWPC repport with two socket and one RAPL_EVENT Scenario: - Launch the full architecture Test if: - each 50 ms, reports are writen in the output database """ # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) t = time.time() number_of_output_reports = 0 for i in range(3): time.sleep(0.2) current = get_number_of_output_reports() assert current >= number_of_output_reports number_of_output_reports = current time.sleep(0.1) supervisor.join()