def launch_simple_architecture(config, supervisor): # Pusher pusher_generator = PusherGenerator() pusher_info = pusher_generator.generate(config) pusher_cls, pusher_start_message = pusher_info['test_pusher'] pusher = supervisor.launch(pusher_cls, pusher_start_message) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher_start_message = DispatcherStartMessage( 'system', 'dispatcher', DummyFormulaActor, DummyFormulaValues({'test_pusher': pusher}, 0), route_table, 'cpu') dispatcher = supervisor.launch(DispatcherActor, dispatcher_start_message) # Puller report_filter = Filter() report_filter.filter(filter_rule, dispatcher) puller_generator = PullerGenerator(report_filter, []) puller_info = puller_generator.generate(config) puller_cls, puller_start_message = puller_info['test_puller'] puller = supervisor.launch(puller_cls, puller_start_message)
def basic_filt(): """ Return a basic filter """ dispatcher = DispatcherActor('dispatcher__', Mock(), Mock()) filt = Filter() filt.filter(lambda msg: True, dispatcher) return filt
def run(self): # Setup signal handler def term_handler(_, __): puller.send_kill() dispatcher.send_kill() pusher.send_kill() exit(0) signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) stream_mode = True supervisor = BackendSupervisor(stream_mode) # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result', HWPCModel()) pusher = PusherActor("pusher_mongodb", PowerReport, output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: RAPLFormulaActor( name, pusher, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep', HWPCModel(), stream_mode=stream_mode) report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) time.sleep(1) os.kill(dispatcher.pid, signal.SIGKILL) supervisor.join()
def run(self): config = { 'verbose': True, 'stream': True, 'output': { 'test_pusher': { 'type': 'mongodb', 'model': 'PowerReport', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_OUTPUT_COLLECTION_NAME } }, 'input': { 'test_puller': { 'type': 'mongodb', 'model': 'HWPCReport', 'uri': MONGO_URI, 'db': MONGO_DATABASE_NAME, 'collection': MONGO_INPUT_COLLECTION_NAME } } } supervisor = Supervisor(config['verbose']) def term_handler(_, __): supervisor.shutdown() exit(0) signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) # Pusher pusher_generator = PusherGenerator() pusher_info = pusher_generator.generate(config) pusher_cls, pusher_start_message = pusher_info['test_pusher'] pusher = supervisor.launch(pusher_cls, pusher_start_message) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher_start_message = DispatcherStartMessage( 'system', 'dispatcher', DummyFormulaActor, DummyFormulaValues({'test_pusher': pusher}, 0), route_table, 'cpu') dispatcher = supervisor.launch(DispatcherActor, dispatcher_start_message) # Puller report_filter = Filter() report_filter.filter(filter_rule, dispatcher) puller_generator = PullerGenerator(report_filter, []) puller_info = puller_generator.generate(config) puller_cls, puller_start_message = puller_info['test_puller'] puller = supervisor.launch(puller_cls, puller_start_message) supervisor.monitor()
def fake_dispatcher_filt(): """ Return a filter that dispatch to a fake dispatcher """ dispatcher = FakeDispatcherActor('fake_dispatcher__' + str(random.randint(1, 100000)), DISPATCHER_SOCKET_ADDRESS, level_logger=LOG_LEVEL) filt = Filter() filt.filter(lambda msg: True, dispatcher) return filt
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor): """ run the same test but set a delay of 1s after sendig the 5th first messages """ # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL, max_size=0) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_socket = SocketDB(unused_tcp_port) report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_socket", input_socket, report_filter, HWPCModel(), level_logger=LOG_LEVEL, stream_mode=True) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) time.sleep(1) json_reports = extract_json_report(10) client = ClientThreadDelay(json_reports, unused_tcp_port) client.start() time.sleep(2) check_db()
def puller(request, database, stream_mode): """ Setup and Teardown for managing a PullerActor setup: create a PullerActor and start its process teardown: terminate the PullerActor process """ dispatcher = DispatcherActor('dispatcher__', Mock(), Mock()) filt = Filter() filt.filter(lambda msg: True, dispatcher) puller_actor = PullerActor("test_puller_" + str(request.node.name), database, filt, HWPCModel(), stream_mode=stream_mode, level_logger=LOG_LEVEL) return puller_actor
def test_run(database, supervisor): # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: CrashFormulaActor( name, {'my_pusher': pusher}, 6, CrashException, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) supervisor.join() check_db()
def test_with_two_filter(self, database): """ Test filter with two filter - 2 first report return first dispatcher - 2 next report return first and second dispatcher - 2 next report return None """ mongodb = MongoDB(URI, "test_filter", "test_filter1") mongodb.connect() hwpc_filter = Filter() hwpc_filter.filter(lambda msg: True if "sensor" in msg.sensor else False, 1) hwpc_filter.filter(lambda msg: True if "test1" in msg.sensor else False, 2) hwpc_filter.filter(lambda msg: True if msg.sensor == "sensor_test2" else False, 3) mongodb_it = mongodb.iter(HWPCModel(), False) for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1, 2] for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1, 3] for _ in range(2): hwpc_report = next(mongodb_it) assert hwpc_filter.route(hwpc_report) == [1]
def test_run(database, supervisor): # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) t = time.time() number_of_output_reports = 0 for i in range(3): time.sleep(0.2) current = get_number_of_output_reports() assert current > number_of_output_reports number_of_output_reports = current time.sleep(0.1) supervisor.join()
def fake_filter(self, fake_dispatcher): fake_filter = Filter() fake_filter.filter(filter_rule, fake_dispatcher) return fake_filter
def launch_powerapi(config, logger): ########################################################################## # Actor Creation # Pusher pushers = generate_pushers(config) # Formula formula_factory = (lambda name, verbose: RAPLFormulaActor( name, pushers, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=config['verbose']) # Puller report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) pullers = generate_pullers(config, report_filter) ########################################################################## # Actor start step # Setup signal handler def term_handler(_, __): for _, puller in pullers.items(): puller.send_kill() dispatcher.send_kill() for _, pusher in pushers.items(): pusher.send_kill() exit(0) signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) supervisor = BackendSupervisor(config['stream']) try: for _, pusher in pushers.items(): supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) for _, puller in pullers.items(): supervisor.launch_actor(puller) except zmq.error.ZMQError as exn: logger.error('Communication error, ZMQError code : ' + str(exn.errno) + ' reason : ' + exn.strerror) supervisor.kill_actors() except ActorInitError as exn: logger.error('Actor initialisation error, reason : ' + exn.message) supervisor.kill_actors() ########################################################################## # Actor run step # wait supervisor.join()
def test_run(files, supervisor): config = { 'verbose': LOG_LEVEL, 'stream': False, 'input': { 'csv': { 'puller': { 'files': FILES, 'model': 'HWPCReport', 'name': 'puller', } } }, 'output': { 'csv': { 'pusher': { 'model': 'PowerReport', 'name': 'pusher', 'directory': ROOT_PATH } } } } # Pusher pusher_generator = PusherGenerator() pushers = pusher_generator.generate(config) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, pushers, level_logger=config['verbose'])) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller_generator = PullerGenerator(report_filter) pullers = puller_generator.generate(config) for _, pusher in pushers.items(): supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) for _, puller in pullers.items(): supervisor.launch_actor(puller) supervisor.join() check_output_file()
def test_run(database, supervisor): """ Test the architecture when it has to deal with a lot of data, pulled with a hight frequency. Input database contain a lot of data, that will overflow the backend. Backend have to handle all of this data and continue to compute data and writing them in real time. Architecture : - 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off) - 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET) - 1 Dummy Formula - 1 pusher (connected to MongoDB1 [collection test_result] MongoDB1 content : - 3000 HWPC repport with two socket and one RAPL_EVENT Scenario: - Launch the full architecture Test if: - each 50 ms, reports are writen in the output database """ # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) t = time.time() number_of_output_reports = 0 for i in range(3): time.sleep(0.2) current = get_number_of_output_reports() assert current >= number_of_output_reports number_of_output_reports = current time.sleep(0.1) supervisor.join()
def test_with_slow_formula(database2, supervisor): """Test the architecture when it has to deal with a lot of data, with a formula that is very slow. Input database contain a lot of data, but formula can't handle them. The puller will end before formula end to handle data and supervisor will begin to send poisonpill message to all actors. Formula and pusher must continue to handle report. Architecture : - 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off) - 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET) - 1 Dummy Formula with will wait 0.2 ms when it receive a message - 1 pusher (connected to MongoDB1 [collection test_result] MongoDB1 content : - 30 HWPC repport with two socket and one RAPL_EVENT Scenario: - Launch the full architecture Test if: - after powerapi handle all the reports, test if all the reports was handeled by the formula and pushed to the mongo database and if no report were lost """ # Pusher output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result') pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, {'my_pusher': pusher}, level_logger=verbose, sleep_time=0.1)) # DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep') report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL) supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) supervisor.launch_actor(puller) supervisor.join() assert get_number_of_output_reports() == 30 * 2
def run_smartwatts(args, logger): """ Run PowerAPI with the SmartWatts formula. :param args: CLI arguments namespace :param logger: Log level to use for the actors """ # Print configuration logger.info('SmartWatts version %s using PowerAPI version %s', smartwatts_version, powerapi_version) logger.info('CPU formula parameters: RAPL_REF=%s ERROR_THRESHOLD=%sW' % (args.cpu_rapl_ref_event, args.cpu_error_threshold)) logger.info('DRAM formula parameters: RAPL_REF=%s ERROR_THRESHOLD=%sW' % (args.dram_rapl_ref_event, args.dram_error_threshold)) # Reports pusher power_output_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_powermeter_collection, None) power_report_pusher = PusherActor('power_report_pusher', PowerReport, power_output_mongodb) formula_output_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_formula_collection, None) formula_report_pusher = PusherActor('formula_report_pusher', FormulaReport, formula_output_mongodb) # Sensor reports route table route_table = RouteTable() route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(HWPCDepthLevel.SOCKET, primary=True)) # Shared parameters pushers = {'power': power_report_pusher, 'formula': formula_report_pusher} cpu_topology = CPUTopology(args.cpu_base_clock, args.cpu_ratio_min, args.cpu_ratio_base, args.cpu_ratio_max) # CPU formula dispatcher def cpu_formula_factory(name: str, _): scope = SmartWattsFormulaScope.CPU config = SmartWattsFormulaConfig(scope, args.cpu_rapl_ref_event, args.cpu_error_threshold, cpu_topology) return SmartWattsFormulaActor(name, pushers, config) cpu_dispatcher = DispatcherActor('cpu_dispatcher', cpu_formula_factory, route_table) # DRAM formula dispatcher def dram_formula_factory(name: str, _): scope = SmartWattsFormulaScope.DRAM config = SmartWattsFormulaConfig(scope, args.cpu_rapl_ref_event, args.cpu_error_threshold, cpu_topology) return SmartWattsFormulaActor(name, pushers, config) dram_dispatcher = DispatcherActor('dram_dispatcher', dram_formula_factory, route_table) # Sensor reports puller input_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_sensor_collection, HWPCModel(), stream_mode=True) report_filter = Filter() report_filter.filter(lambda msg: True, cpu_dispatcher) report_filter.filter(lambda msg: True, dram_dispatcher) puller = PullerActor('hwpc_report_puller', input_mongodb, report_filter) def term_handler(_, __): puller.join() cpu_dispatcher.join() dram_dispatcher.join() power_report_pusher.join() formula_report_pusher.join() exit(0) # TERM/INT signals handler signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) # Actors supervision supervisor = Supervisor() try: supervisor.launch_actor(power_report_pusher) supervisor.launch_actor(formula_report_pusher) supervisor.launch_actor(cpu_dispatcher) supervisor.launch_actor(dram_dispatcher) supervisor.launch_actor(puller) logger.info('Actors initialized, SmartWatts is now running...') except zmq.error.ZMQError as exn: logger.error('Communication error, ZMQError code : ' + str(exn.errno) + ' reason : ' + exn.strerror) supervisor.kill_actors() except ActorInitError as exn: logger.error('Actor initialisation error, reason : ' + exn.message) supervisor.kill_actors() supervisor.join()