def test_no_input_specified(mocked_sys_exit): """ generate puller from an empty config dict Test if the generate_puller function call sys.exit """ args = {} # sys.exit = Mock() # sys.exit.side_effect = Exception() generator = PullerGenerator(None) with pytest.raises(SysExitException): generator.generate(args)
def test_no_input_specified(): """ generate puller from an empty config dict Test if the generate_puller function call sys.exit """ args = {} sys.exit = Mock() sys.exit.side_effect = Exception() generator = PullerGenerator(None) with pytest.raises(Exception): generator.generate(args) assert sys.exit.called
def test_generate_puller_from_simple_config(): """ generate mongodb puller from this config : { 'verbose': True, 'stream': True, 'input': {'mongodb': {'toto': {'model': 'HWPCReport', 'name': 'toto', 'uri': 'titi', 'db': 'tata', 'collection': 'tutu'}}}} Test if : - function return a dict containing one actor, its key is 'toto' - puller type is PullerActor - puller name is toto - puller database type is MongoDB - database uri is titi - database db is tata - database collection is tutu """ args = { 'verbose': True, 'stream': True, 'input': { 'mongodb': { 'toto': { 'model': 'HWPCReport', 'name': 'toto', 'uri': 'titi', 'db': 'tata', 'collection': 'tutu' } } } } generator = PullerGenerator(None) result = generator.generate(args) assert len(result) == 1 assert 'toto' in result puller = result['toto'] assert isinstance(puller, PullerActor) assert puller.name == 'toto' db = puller.state.database assert isinstance(db, MongoDB) assert db.uri == 'titi' assert db.db_name == 'tata' assert db.collection_name == 'tutu'
def test_remove_mongodb_factory_and_generate_puller_from_a_config_with_mongodb_input_must_call_sys_exit_( mocked_sys_exit): args = { 'verbose': True, 'stream': True, 'input': { 'mongodb': { 'toto': { 'model': 'HWPCReport', 'name': 'toto', 'uri': 'titi', 'db': 'tata', 'collection': 'tutu' } } } } generator = PullerGenerator(None) generator.remove_db_factory('mongodb') with pytest.raises(SysExitException): result = generator.generate(args)
def test_run(files, supervisor): config = { 'verbose': LOG_LEVEL, 'stream': False, 'input': { 'csv': { 'puller': { 'files': FILES, 'model': 'HWPCReport', 'name': 'puller', } } }, 'output': { 'csv': { 'pusher': { 'model': 'PowerReport', 'name': 'pusher', 'directory': ROOT_PATH } } } } # Pusher pusher_generator = PusherGenerator() pushers = pusher_generator.generate(config) # Formula formula_factory = (lambda name, verbose: DummyFormulaActor( name, pushers, level_logger=config['verbose'])) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL) # Puller report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller_generator = PullerGenerator(report_filter) pullers = puller_generator.generate(config) for _, pusher in pushers.items(): supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) for _, puller in pullers.items(): supervisor.launch_actor(puller) supervisor.join() check_output_file()
def test_generate_two_pusher(): """ generate two mongodb puller from this config : { 'verbose': True, 'stream': True, 'input': {'mongodb': {'toto': {'model': 'HWPCReport', 'name': 'toto', 'uri': 'titi', 'db': 'tata', 'collection': 'tutu'}, 'titi': {'model': 'HWPCReport', 'name': 'titi', 'uri': 'titi', 'db': 'tata', 'collection': 'huhu'}}}} Test if : - function return a dict containing two actor, their key are 'toto' and 'titi' - pullers type are PullerActor - pullers name are toto and titi - pullers database type are MongoDB - databases uri are titi - databases db are tata - databases collection are tutu and huhu """ args = { 'verbose': True, 'stream': True, 'input': { 'mongodb': { 'toto': { 'model': 'HWPCReport', 'name': 'toto', 'uri': 'titi', 'db': 'tata', 'collection': 'tutu' }, 'titi': { 'model': 'HWPCReport', 'name': 'titi', 'uri': 'titi', 'db': 'tata', 'collection': 'huhu' } } } } generator = PullerGenerator(None) result = generator.generate(args) assert len(result) == 2 assert 'toto' in result puller = result['toto'] assert isinstance(puller, PullerActor) assert puller.name == 'toto' db = puller.state.database assert isinstance(db, MongoDB) assert db.uri == 'titi' assert db.db_name == 'tata' assert db.collection_name == 'tutu' assert 'titi' in result puller = result['titi'] assert isinstance(puller, PullerActor) assert puller.name == 'titi' db = puller.state.database assert isinstance(db, MongoDB) assert db.uri == 'titi' assert db.db_name == 'tata' assert db.collection_name == 'huhu'
def launch_powerapi(config, logger): # Pusher pusher_generator = PusherGenerator() pushers = pusher_generator.generate(config) # Formula formula_factory = (lambda name, verbose: RAPLFormulaActor( name, pushers, level_logger=verbose)) # Dispatcher route_table = RouteTable() route_table.dispatch_rule( HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True)) dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=config['verbose']) # Puller report_filter = Filter() report_filter.filter(lambda msg: True, dispatcher) puller_generator = PullerGenerator(report_filter) pullers = puller_generator.generate(config) # Setup signal handler def term_handler(_, __): for _, puller in pullers.items(): puller.send_kill() dispatcher.send_kill() for _, pusher in pushers.items(): pusher.send_kill() exit(0) signal.signal(signal.SIGTERM, term_handler) signal.signal(signal.SIGINT, term_handler) supervisor = BackendSupervisor(config['stream']) try: for _, pusher in pushers.items(): supervisor.launch_actor(pusher) supervisor.launch_actor(dispatcher) for _, puller in pullers.items(): supervisor.launch_actor(puller) except zmq.error.ZMQError as exn: logger.error('Communication error, ZMQError code : ' + str(exn.errno) + ' reason : ' + exn.strerror) supervisor.kill_actors() except ActorInitError as exn: logger.error('Actor initialisation error, reason : ' + exn.message) supervisor.kill_actors() # wait supervisor.join()