def test_with_two_filter(self, database):
"""
Test filter with two filter
- 2 first report return first dispatcher
- 2 next report return first and second dispatcher
- 2 next report return None
"""
mongodb = MongoDB(URI, "test_filter", "test_filter1")
mongodb.connect()
hwpc_filter = Filter()
hwpc_filter.filter(lambda msg:
True if "sensor" in msg.sensor else False,
1)
hwpc_filter.filter(lambda msg:
True if "test1" in msg.sensor else False,
2)
hwpc_filter.filter(lambda msg:
True if msg.sensor == "sensor_test2" else False,
3)
mongodb_it = mongodb.iter(HWPCModel(), False)
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1, 2]
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1, 3]
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1]
def test_mongodb_read_capped_db(database):
"""
Test read mongodb capped collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb",
"test_mongodb2", HWPCModel())
# Check if we can read one time
mongodb.connect()
mongodb_iter = iter(mongodb)
for _ in range(mongodb.collection.count_documents({})):
report = next(mongodb_iter)
assert report is not None
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
# Add data in the collection
for _ in range(2):
report.pop('_id', None)
mongodb.collection.insert_one(report)
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def run(self):
# Setup signal handler
def term_handler(_, __):
puller.send_kill()
dispatcher.send_kill()
pusher.send_kill()
exit(0)
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
stream_mode = True
supervisor = BackendSupervisor(stream_mode)
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result',
HWPCModel())
pusher = PusherActor("pusher_mongodb",
PowerReport,
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: RAPLFormulaActor(
name, pusher, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI,
'MongoDB1',
'test_hwrep',
HWPCModel(),
stream_mode=stream_mode)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
os.kill(dispatcher.pid, signal.SIGKILL)
supervisor.join()
def test_mongodb_bad_port(database):
"""
Test if the database doesn't exist (hostname/port error)
"""
with pytest.raises(MongoBadDBError) as pytest_wrapped:
MongoDB("mongodb://localhost:1", "error", "error").connect()
assert pytest_wrapped.type == MongoBadDBError
def test_mongodb_bad_hostname(mongo_database):
"""
Test if the database doesn't exist (hostname/port error)
"""
with pytest.raises(MongoBadDBError) as pytest_wrapped:
MongoDB(HWPCReport, "mongodb://lel:27017/", "error", "error").connect()
assert pytest_wrapped.type == MongoBadDBError
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config: SocketDB(db_config['port']),
'csv':
lambda db_config: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
}
def test_mongodb_all_reports(database):
"""
Test create/save/read all kind of reports
"""
all_reports = [(HWPCModel(), gen_hwpc_report),
(PowerModel(), gen_power_report)]
for model, generator in all_reports:
# Load DB
mongodb = MongoDB(URI, "test_reports" + model.get_type().__name__,
"test_reports" + model.get_type().__name__)
mongodb.connect()
# Create report
report = generator()
# Save report
mongodb.save(report, model)
# Read report
mongodb_iter = mongodb.iter(model, False)
read_report = next(mongodb_iter)
# Compare
assert read_report == report
def test_mongodb_read_capped_db(mongo_database):
"""
Test read mongodb capped collection
"""
# Load DB
mongodb = MongoDB(HWPCReport, MONGO_URI, MONGO_DATABASE_NAME,
MONGO_INPUT_COLLECTION_NAME)
# Check if we can read one time
mongodb.connect()
mongodb_iter = mongodb.iter(False)
report = None
for _ in range(mongodb.collection.count_documents({})):
report = next(mongodb_iter)
assert report is not None
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
# Add data in the collection
for _ in range(1):
mongodb.save(report)
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def test_mongodb_read_basic_db(database):
"""
Test read mongodb collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb", "test_mongodb1")
# Check if we can reload after reading
mongodb.connect()
for _ in range(2):
mongodb_iter = mongodb.iter(HWPCModel(), False)
for _ in range(10):
assert next(mongodb_iter) is not None
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def mongodb_database(uri, database_name, collection_name, stream_mode):
"""
Return MongoDB database
"""
database = MongoDB(uri,
database_name,
collection_name,
HWPCModel(),
stream_mode=stream_mode)
return database
def test_mongodb_read_basic_db(mongo_database):
"""
Test read mongodb collection
"""
# Load DB
mongodb = MongoDB(HWPCReport, MONGO_URI, MONGO_DATABASE_NAME,
MONGO_INPUT_COLLECTION_NAME)
# Check if we can reload after reading
mongodb.connect()
for _ in range(2):
mongodb_iter = mongodb.iter(False)
for _ in range(10):
assert next(mongodb_iter) is not None
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: CrashFormulaActor(
name, {'my_pusher': pusher}, 6, CrashException, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
check_db()
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose:
DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current > number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor):
"""
run the same test but set a delay of 1s after sendig the 5th first messages
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL,
max_size=0)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_socket = SocketDB(unused_tcp_port)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_socket",
input_socket,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL,
stream_mode=True)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
json_reports = extract_json_report(10)
client = ClientThreadDelay(json_reports, unused_tcp_port)
client.start()
time.sleep(2)
check_db()
def test_mongodb_save_basic_db(database):
"""
Test save mongodb collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb", "test_mongodb3")
mongodb.connect()
# Check if save work
basic_count = mongodb.collection.count_documents({})
for _ in range(2):
mongodb.save(gen_hwpc_report(), HWPCModel())
assert mongodb.collection.count_documents({}) == basic_count + 2
def test_mongodb_save_basic_db(mongo_database):
"""
Test save mongodb collection
"""
# Load DB
mongodb = MongoDB(HWPCReport, MONGO_URI, MONGO_DATABASE_NAME,
MONGO_INPUT_COLLECTION_NAME)
mongodb.connect()
# Check if save work
basic_count = mongodb.collection.count_documents({})
for report in gen_HWPCReports(2):
mongodb.save(report)
assert mongodb.collection.count_documents({}) == basic_count + 2
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config, _: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config, _: SocketDB(db_config['port']),
'csv':
lambda db_config, _: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config, _: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config, _: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
'prom':
lambda db_config, _: PrometheusDB(
db_config['port'], db_config['addr'], db_config['metric_name'],
db_config['metric_description'], self.model_factory[db_config[
'model']], db_config['aggregation_period']),
'tcp':
lambda db_config, group_name: IOTcpDB(db_config['uri'],
db_config['port'],
input=(group_name == "input")
)
}
def run_smartwatts(args, logger):
"""
Run PowerAPI with the SmartWatts formula.
:param args: CLI arguments namespace
:param logger: Log level to use for the actors
"""
# Print configuration
logger.info('SmartWatts version %s using PowerAPI version %s', smartwatts_version, powerapi_version)
logger.info('CPU formula parameters: RAPL_REF=%s ERROR_THRESHOLD=%sW' % (args.cpu_rapl_ref_event, args.cpu_error_threshold))
logger.info('DRAM formula parameters: RAPL_REF=%s ERROR_THRESHOLD=%sW' % (args.dram_rapl_ref_event, args.dram_error_threshold))
# Reports pusher
power_output_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_powermeter_collection, None)
power_report_pusher = PusherActor('power_report_pusher', PowerReport, power_output_mongodb)
formula_output_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_formula_collection, None)
formula_report_pusher = PusherActor('formula_report_pusher', FormulaReport, formula_output_mongodb)
# Sensor reports route table
route_table = RouteTable()
route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(HWPCDepthLevel.SOCKET, primary=True))
# Shared parameters
pushers = {'power': power_report_pusher, 'formula': formula_report_pusher}
cpu_topology = CPUTopology(args.cpu_base_clock, args.cpu_ratio_min, args.cpu_ratio_base, args.cpu_ratio_max)
# CPU formula dispatcher
def cpu_formula_factory(name: str, _):
scope = SmartWattsFormulaScope.CPU
config = SmartWattsFormulaConfig(scope, args.cpu_rapl_ref_event, args.cpu_error_threshold, cpu_topology)
return SmartWattsFormulaActor(name, pushers, config)
cpu_dispatcher = DispatcherActor('cpu_dispatcher', cpu_formula_factory, route_table)
# DRAM formula dispatcher
def dram_formula_factory(name: str, _):
scope = SmartWattsFormulaScope.DRAM
config = SmartWattsFormulaConfig(scope, args.cpu_rapl_ref_event, args.cpu_error_threshold, cpu_topology)
return SmartWattsFormulaActor(name, pushers, config)
dram_dispatcher = DispatcherActor('dram_dispatcher', dram_formula_factory, route_table)
# Sensor reports puller
input_mongodb = MongoDB(args.mongodb_uri, args.mongodb_database, args.mongodb_sensor_collection, HWPCModel(), stream_mode=True)
report_filter = Filter()
report_filter.filter(lambda msg: True, cpu_dispatcher)
report_filter.filter(lambda msg: True, dram_dispatcher)
puller = PullerActor('hwpc_report_puller', input_mongodb, report_filter)
def term_handler(_, __):
puller.join()
cpu_dispatcher.join()
dram_dispatcher.join()
power_report_pusher.join()
formula_report_pusher.join()
exit(0)
# TERM/INT signals handler
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
# Actors supervision
supervisor = Supervisor()
try:
supervisor.launch_actor(power_report_pusher)
supervisor.launch_actor(formula_report_pusher)
supervisor.launch_actor(cpu_dispatcher)
supervisor.launch_actor(dram_dispatcher)
supervisor.launch_actor(puller)
logger.info('Actors initialized, SmartWatts is now running...')
except zmq.error.ZMQError as exn:
logger.error('Communication error, ZMQError code : ' + str(exn.errno) + ' reason : ' + exn.strerror)
supervisor.kill_actors()
except ActorInitError as exn:
logger.error('Actor initialisation error, reason : ' + exn.message)
supervisor.kill_actors()
supervisor.join()
def mongodb_database(uri, database_name, collection_name):
"""
Return MongoDB database
"""
database = MongoDB(uri, database_name, collection_name)
return database
def test_with_slow_formula(database2, supervisor):
"""Test the architecture when it has to deal with a lot of data, with a formula
that is very slow. Input database contain a lot of data, but formula can't
handle them. The puller will end before formula end to handle data and
supervisor will begin to send poisonpill message to all actors. Formula and
pusher must continue to handle report.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula with will wait 0.2 ms when it receive a message
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 30 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- after powerapi handle all the reports, test if all the reports was handeled
by the formula and pushed to the mongo database and if no report were lost
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose, sleep_time=0.1))
# DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
assert get_number_of_output_reports() == 30 * 2
print(msg, file=sys.stderr)
except UnknowArgException as exn:
msg = 'CLI error : unknow argument ' + exn.argument_name
print(msg, file=sys.stderr)
except BadContextException as exn:
msg = 'CLI error : argument ' + exn.argument_name
msg += 'not used in the correct context\nUse it with the following arguments :'
for main_arg_name, context_name in exn.context_list:
msg += '\n --' + main_arg_name + ' ' + context_name
print(msg, file=sys.stderr)
sys.exit()
DB_FACTORY = {
'mongodb':
lambda db_config: MongoDB(db_config['uri'], db_config['db'], db_config[
'collection']),
}
MODEL_FACTORY = {
'hwpc_report': HWPCModel(),
'power_report': PowerModel(),
}
def generate_pullers(config, report_filter):
pullers = {}
for db_config in config['input']:
try:
factory = DB_FACTORY[db_config['type']]
def test_run(database, supervisor):
"""
Test the architecture when it has to deal with a lot of data, pulled with a
hight frequency. Input database contain a lot of data, that will overflow the
backend. Backend have to handle all of this data and continue to compute data
and writing them in real time.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 3000 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- each 50 ms, reports are writen in the output database
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current >= number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()