def run(self):
# Setup signal handler
def term_handler(_, __):
puller.hard_kill()
dispatcher.hard_kill()
pusher.hard_kill()
exit(0)
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
stream_mode = True
supervisor = BackendSupervisor(stream_mode)
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'id_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
stream_mode=stream_mode,
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
os.kill(dispatcher.pid, signal.SIGKILL)
supervisor.join()
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor):
"""
run the same test but set a delay of 1s after sendig the 5th first messages
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL,
max_size=0)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_socket = SocketDB(unused_tcp_port)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_socket",
input_socket,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL,
stream_mode=True)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
json_reports = extract_json_report(10)
client = ClientThreadDelay(json_reports, unused_tcp_port)
client.start()
time.sleep(2)
check_db()
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
check_db()
def test_run(files, supervisor):
config = {
'verbose': LOG_LEVEL,
'stream': False,
'input': {
'csv': {
'puller': {
'files': FILES,
'model': 'HWPCReport',
'name': 'puller',
}
}
},
'output': {
'csv': {
'pusher': {
'model': 'PowerReport',
'name': 'pusher',
'directory': ROOT_PATH
}
}
}
}
# Pusher
pusher_generator = PusherGenerator()
pushers = pusher_generator.generate(config)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, pushers, level_logger=config['verbose']))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller_generator = PullerGenerator(report_filter)
pullers = puller_generator.generate(config)
for _, pusher in pushers.items():
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
for _, puller in pullers.items():
supervisor.launch_actor(puller)
supervisor.join()
check_output_file()
def test_run(database, supervisor):
"""
Test the architecture when it has to deal with a lot of data, pulled with a
hight frequency. Input database contain a lot of data, that will overflow the
backend. Backend have to handle all of this data and continue to compute data
and writing them in real time.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 3000 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- each 50 ms, reports are writen in the output database
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current >= number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
def test_with_slow_formula(database2, supervisor):
"""Test the architecture when it has to deal with a lot of data, with a formula
that is very slow. Input database contain a lot of data, but formula can't
handle them. The puller will end before formula end to handle data and
supervisor will begin to send poisonpill message to all actors. Formula and
pusher must continue to handle report.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula with will wait 0.2 ms when it receive a message
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 30 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- after powerapi handle all the reports, test if all the reports was handeled
by the formula and pushed to the mongo database and if no report were lost
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose, sleep_time=0.1))
# DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
assert get_number_of_output_reports() == 30 * 2