def test_csvdb_save_on(self, clean_csv_files):
"""
Save a PowerReport from an basic object
"""
csvdb = CsvDB(current_path=PATH_TO_SAVE)
csvdb.connect()
power_reports = list()
# Save one time with a file that doesn't exist
power_reports.append(gen_power_report())
csvdb.save(power_reports[0], PowerModel())
# Save three time
for _ in range(3):
power_reports.append(gen_power_report())
csvdb.save(power_reports[-1], PowerModel())
# Read the the csvdb and compare the data
reading_power_reports = []
csvdb_read = CsvDB(current_path=PATH_TO_SAVE)
csvdb_read.add_file(PATH_TO_SAVE + SENSOR + "-" + TARGET + "/PowerReport.csv")
csvdb_read.connect()
csvdb_read_iter = csvdb_read.iter(PowerModel(), False)
for _ in range(4):
reading_power_reports.append(next(csvdb_read_iter))
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_read_iter)
assert pytest_wrapped.type == StopIteration
for i in range(4):
assert power_reports[i] == reading_power_reports[i]
def test_convert_to_influxDB_format_dont_keep_predict_in_tag():
serialized_report = {
'sensor': 'sensor_name',
'target': 'target_name',
'timestamp': datetime.datetime.now().timestamp(),
'power': 3,
'metadata': {
'scope': 2,
'socket': 1,
'formula': 'formuna_name',
'predict': 3}
}
model = PowerModel()
influxdb_format = model.to_influxdb(serialized_report)
assert 'predict' not in influxdb_format['tags']
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config: SocketDB(db_config['port']),
'csv':
lambda db_config: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
}
def test_mongodb_all_reports(database):
"""
Test create/save/read all kind of reports
"""
all_reports = [(HWPCModel(), gen_hwpc_report),
(PowerModel(), gen_power_report)]
for model, generator in all_reports:
# Load DB
mongodb = MongoDB(URI, "test_reports" + model.get_type().__name__,
"test_reports" + model.get_type().__name__)
mongodb.connect()
# Create report
report = generator()
# Save report
mongodb.save(report, model)
# Read report
mongodb_iter = mongodb.iter(model, False)
read_report = next(mongodb_iter)
# Compare
assert read_report == report
def test_csvdb_all_reports(clean_csv_files):
"""
Test create/save/read all kind of reports
"""
all_reports = [(PowerModel(), gen_power_report),
(HWPCModel(), gen_hwpc_report)]
for model, generator in all_reports:
# Load DB
csvdb = CsvDB(current_path=PATH_TO_SAVE)
csvdb.connect()
# Create report
report = generator()
# Save report
csvdb.save(report, model)
# Read report
for r, d, f in os.walk(PATH_TO_SAVE + report.sensor + "-" + report.target + "/"):
for file in f:
if '.csv' in file:
csvdb.add_file(os.path.join(r, file))
csvdb_iter = csvdb.iter(model, False)
read_report = next(csvdb_iter)
# Compare
assert read_report == report
def test_csvdb_bad_filepath(self, csvdb):
"""
Test with bad filepath
"""
with pytest.raises(CsvBadFilePathError) as pytest_wrapped:
csvdb.add_file("/tmp/unknowfile.csv")
csvdb.connect()
csvdb.iter(PowerModel(), False)
assert pytest_wrapped.type == CsvBadFilePathError
def test_write_many_report_in_non_empty_db(init_database_with_one_report, database):
"""
call the save_many method with One PowerReport
test if the report was writen in the database
"""
database.connect()
database.save_many([POWER_REPORT_1, POWER_REPORT_2], PowerModel())
check_db_reports(init_database_with_one_report, [POWER_REPORT_0, POWER_REPORT_1, POWER_REPORT_2])
def test_write_one_report_in_empty_db(init_empty_database, database):
"""
call the save method with One PowerReport
test if the report was writen in the database
"""
database.connect()
database.save(POWER_REPORT_1, PowerModel())
check_db_reports(init_empty_database, [POWER_REPORT_1])
def run(self):
# Setup signal handler
def term_handler(_, __):
puller.hard_kill()
dispatcher.hard_kill()
pusher.hard_kill()
exit(0)
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
stream_mode = True
supervisor = BackendSupervisor(stream_mode)
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'id_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
stream_mode=stream_mode,
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
os.kill(dispatcher.pid, signal.SIGKILL)
supervisor.join()
def test_write_one_bad_report_in_empty_db(init_empty_database, database):
"""
call the save method with One PowerReport with no sensor field
test if a BadInputData is raise and the database is still empty
"""
database.connect()
with pytest.raises(BadInputData):
database.save(BAD_POWER_REPORT, PowerModel())
assert get_all_reports(init_empty_database, INFLUX_DBNAME) == []
def test_write_one_bad_report_in_non_empty_db(init_database_with_one_report, database):
"""
call the save method with One PowerReport with no sensor field
test if a BadInputData is raise and the database is still empty
"""
database.connect()
with pytest.raises(BadInputData):
database.save(BAD_POWER_REPORT, PowerModel())
check_db_reports(init_database_with_one_report, [POWER_REPORT_0])
def test_csvdb_save_header_error(self, corrupted_csvdb):
"""
Try to save a PowerReport with an existent file and a corrupted header
"""
csvdb = CsvDB(current_path=PATH_TO_SAVE)
csvdb.connect()
# Try to save one PowerReport
power_report = gen_power_report()
with pytest.raises(HeaderAreNotTheSameError) as pytest_wrapped:
csvdb.save(power_report, PowerModel())
assert pytest_wrapped.type == HeaderAreNotTheSameError
def test_csvdb_bad_common(self, csvdb):
"""
Test when file miss some common column
"""
csv_files = BAD_COMMON
while csv_files:
with pytest.raises(CsvBadCommonKeysError) as pytest_wrapped:
csvdb.add_files(csv_files)
csvdb.connect()
csvdb.iter(PowerModel(), False)
assert pytest_wrapped.type == CsvBadCommonKeysError
csv_files = csv_files[1:]
def test_one_good_one_bad_reports_in_non_empty_db(init_database_with_one_report, database):
"""
call the save_many method on a list composed with one good formated power
report and one bad formated power report (missing sensor field)
test if a BadInputData is raise and the database is still empty
"""
database.connect()
with pytest.raises(BadInputData):
database.save_many([POWER_REPORT_1, BAD_POWER_REPORT], PowerModel())
check_db_reports(init_database_with_one_report, [POWER_REPORT_0])
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor):
"""
run the same test but set a delay of 1s after sendig the 5th first messages
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL,
max_size=0)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_socket = SocketDB(unused_tcp_port)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_socket",
input_socket,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL,
stream_mode=True)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
json_reports = extract_json_report(10)
client = ClientThreadDelay(json_reports, unused_tcp_port)
client.start()
time.sleep(2)
check_db()
def test_pusher_save_power_report(generate_mongodb_data, initialized_pusher):
"""
Create a PusherActor, send him a PowerReport, kill him and check it from database
"""
# Connect data and send a PowerReport
initialized_pusher.connect_data()
saved_report = gen_power_report()
initialized_pusher.send_data(saved_report)
# Kill it
initialized_pusher.send_kill(by_data=True)
assert not is_actor_alive(initialized_pusher, 5)
# Open a database for read the saved report
mongodb = initialized_pusher.state.database
mongodb.connect()
mongodb_iter = mongodb.iter(PowerModel(), False)
new_report = next(mongodb_iter)
assert saved_report == new_report
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: CrashFormulaActor(
name, {'my_pusher': pusher}, 6, CrashException, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
check_db()
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config, _: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config, _: SocketDB(db_config['port']),
'csv':
lambda db_config, _: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config, _: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config, _: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
'prom':
lambda db_config, _: PrometheusDB(
db_config['port'], db_config['addr'], db_config['metric_name'],
db_config['metric_description'], self.model_factory[db_config[
'model']], db_config['aggregation_period']),
'tcp':
lambda db_config, group_name: IOTcpDB(db_config['uri'],
db_config['port'],
input=(group_name == "input")
)
}
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose:
DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current > number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
msg += 'not used in the correct context\nUse it with the following arguments :'
for main_arg_name, context_name in exn.context_list:
msg += '\n --' + main_arg_name + ' ' + context_name
print(msg, file=sys.stderr)
sys.exit()
DB_FACTORY = {
'mongodb':
lambda db_config: MongoDB(db_config['uri'], db_config['db'], db_config[
'collection']),
}
MODEL_FACTORY = {
'hwpc_report': HWPCModel(),
'power_report': PowerModel(),
}
def generate_pullers(config, report_filter):
pullers = {}
for db_config in config['input']:
try:
factory = DB_FACTORY[db_config['type']]
model = MODEL_FACTORY[db_config['model']]
name = 'puller_' + db_config['type']
puller = PullerActor(name,
factory(db_config),
report_filter,
def test_run(database, supervisor):
"""
Test the architecture when it has to deal with a lot of data, pulled with a
hight frequency. Input database contain a lot of data, that will overflow the
backend. Backend have to handle all of this data and continue to compute data
and writing them in real time.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 3000 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- each 50 ms, reports are writen in the output database
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current >= number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
def _influxdb_keept_metadata(self):
return PowerModel._influxdb_keept_metadata(self) + ('formula_name',)
def test_with_slow_formula(database2, supervisor):
"""Test the architecture when it has to deal with a lot of data, with a formula
that is very slow. Input database contain a lot of data, but formula can't
handle them. The puller will end before formula end to handle data and
supervisor will begin to send poisonpill message to all actors. Formula and
pusher must continue to handle report.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula with will wait 0.2 ms when it receive a message
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 30 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- after powerapi handle all the reports, test if all the reports was handeled
by the formula and pushed to the mongo database and if no report were lost
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose, sleep_time=0.1))
# DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
assert get_number_of_output_reports() == 30 * 2
])
else:
metafunc.parametrize('database', [database])
if 'report_model' in metafunc.fixturenames:
report_model = getattr(metafunc.function, '_report_model', None)
metafunc.parametrize('report_model', [report_model])
##############################################################################
# Tests #
##############################################################################
@define_database(mongodb_database(URI, "test_mongodb", "test_mongodb1"))
@define_report_model(PowerModel())
def test_pusher_create_ok(started_pusher):
"""
Create a PusherActor with a good configuration
"""
assert is_actor_alive(started_pusher)
@define_database([
("mongodb://toto:27017", "test_mongodb", "test_mongodb1"),
("mongodb://localhost:27016", "test_mongodb", "test_mongodb1"),
])
@define_report_model(PowerModel())
def test_pusher_create_bad_db(pusher, supervisor):
"""
Create a PusherActor with a bad database