def test_mongodb_read_capped_db(database):
"""
Test read mongodb capped collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb", "test_mongodb2")
# Check if we can read one time
mongodb.connect()
mongodb_iter = mongodb.iter(HWPCModel(), False)
report = None
for _ in range(mongodb.collection.count_documents({})):
report = next(mongodb_iter)
assert report is not None
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
# Add data in the collection
for _ in range(1):
mongodb.save(report, HWPCModel())
# Check if there is nothing after
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def run(self):
# Setup signal handler
def term_handler(_, __):
puller.send_kill()
dispatcher.send_kill()
pusher.send_kill()
exit(0)
signal.signal(signal.SIGTERM, term_handler)
signal.signal(signal.SIGINT, term_handler)
stream_mode = True
supervisor = BackendSupervisor(stream_mode)
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result',
HWPCModel())
pusher = PusherActor("pusher_mongodb",
PowerReport,
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: RAPLFormulaActor(
name, pusher, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'ROOT'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI,
'MongoDB1',
'test_hwrep',
HWPCModel(),
stream_mode=stream_mode)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
os.kill(dispatcher.pid, signal.SIGKILL)
supervisor.join()
def test_puller_start_handler(self):
"""
Test the StartHandler of PullerActor
"""
# Define PullerState
fake_database = get_fake_db()
fake_socket_interface = get_fake_socket_interface()
fake_filter = get_fake_filter()
puller_state = PullerState(Mock(), fake_database, fake_filter,
HWPCModel(), False, 100)
puller_state.actor.receive_control = Mock(return_value=None)
assert puller_state.initialized is False
# Define StartHandler
start_handler = PullerStartHandler(puller_state, 0.1)
# Test Random message when state is not initialized
to_send = [OKMessage(), ErrorMessage("Error"), create_report_root({})]
for msg in to_send:
start_handler.handle(msg)
assert fake_database.method_calls == []
assert fake_socket_interface.method_calls == []
assert fake_filter.method_calls == []
assert puller_state.initialized is False
# Try to initialize the state
start_handler.handle(StartMessage())
assert puller_state.initialized is True
def test_csvdb_all_reports(clean_csv_files):
"""
Test create/save/read all kind of reports
"""
all_reports = [(PowerModel(), gen_power_report),
(HWPCModel(), gen_hwpc_report)]
for model, generator in all_reports:
# Load DB
csvdb = CsvDB(current_path=PATH_TO_SAVE)
csvdb.connect()
# Create report
report = generator()
# Save report
csvdb.save(report, model)
# Read report
for r, d, f in os.walk(PATH_TO_SAVE + report.sensor + "-" + report.target + "/"):
for file in f:
if '.csv' in file:
csvdb.add_file(os.path.join(r, file))
csvdb_iter = csvdb.iter(model, False)
read_report = next(csvdb_iter)
# Compare
assert read_report == report
def test_ReportHandler_message_saving_order():
"""
Handle 3 HWPCReport with a pusher.ReportHandler
The maximum size of the handler buffer is 2
This 3 reports are not sent in their chronological order
First report : timestamp = 0
Second report : timestamp = 2
Third report : timestamp = 1
When the handler save the received reports test if the reports are saved in
their chronological order
"""
fake_database = get_fake_db()
fake_database.save_many = Mock()
pusher_state = PusherState(Mock(), fake_database, HWPCModel())
report_handler = ReportHandler(pusher_state, delay=10000, max_size=2)
report0 = HWPCReport(datetime.fromtimestamp(0), None, None, None)
report1 = HWPCReport(datetime.fromtimestamp(2), None, None, None)
report2 = HWPCReport(datetime.fromtimestamp(1), None, None, None)
report_handler.handle(report0)
report_handler.handle(report1)
report_handler.handle(report2)
buffer = fake_database.save_many.call_args[0][0]
assert len(buffer) == 3
assert buffer[0].timestamp < buffer[1].timestamp
assert buffer[1].timestamp < buffer[2].timestamp
def test_with_two_filter(self, database):
"""
Test filter with two filter
- 2 first report return first dispatcher
- 2 next report return first and second dispatcher
- 2 next report return None
"""
mongodb = MongoDB(URI, "test_filter", "test_filter1")
mongodb.connect()
hwpc_filter = Filter()
hwpc_filter.filter(lambda msg:
True if "sensor" in msg.sensor else False,
1)
hwpc_filter.filter(lambda msg:
True if "test1" in msg.sensor else False,
2)
hwpc_filter.filter(lambda msg:
True if msg.sensor == "sensor_test2" else False,
3)
mongodb_it = mongodb.iter(HWPCModel(), False)
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1, 2]
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1, 3]
for _ in range(2):
hwpc_report = next(mongodb_it)
assert hwpc_filter.route(hwpc_report) == [1]
def test_csvdb_bad_filepath(self):
"""
Test with bad filepath
"""
with pytest.raises(CsvBadFilePathError) as pytest_wrapped:
CsvDB(HWPCModel(), ["/tmp/unknowfile"]).connect()
assert pytest_wrapped.type == CsvBadFilePathError
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config: SocketDB(db_config['port']),
'csv':
lambda db_config: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
}
def test_pusher_start_handler(self):
"""
Test the StartHandler of PusherActor
"""
# Define PusherState
fake_database = get_fake_db()
fake_socket_interface = get_fake_socket_interface()
pusher_state = PusherState(Mock(), fake_database, HWPCModel())
pusher_state.actor.socket_interface = fake_socket_interface
assert pusher_state.initialized is False
# Define StartHandler
start_handler = PusherStartHandler(pusher_state)
# Test Random message when state is not initialized
to_send = [OKMessage(), ErrorMessage("Error"), create_report_root({})]
for msg in to_send:
start_handler.handle(msg)
assert fake_database.method_calls == []
assert fake_socket_interface.method_calls == []
assert pusher_state.initialized is False
# Try to initialize the state
start_handler.handle(StartMessage())
assert pusher_state.initialized is True
def test_mongodb_all_reports(database):
"""
Test create/save/read all kind of reports
"""
all_reports = [(HWPCModel(), gen_hwpc_report),
(PowerModel(), gen_power_report)]
for model, generator in all_reports:
# Load DB
mongodb = MongoDB(URI, "test_reports" + model.get_type().__name__,
"test_reports" + model.get_type().__name__)
mongodb.connect()
# Create report
report = generator()
# Save report
mongodb.save(report, model)
# Read report
mongodb_iter = mongodb.iter(model, False)
read_report = next(mongodb_iter)
# Compare
assert read_report == report
def test_mongodb_bad_port(database):
"""
Test if the database doesn't exist (hostname/port error)
"""
with pytest.raises(MongoBadDBError) as pytest_wrapped:
MongoDB("mongodb://localhost:1", "error", "error",
HWPCModel()).connect()
assert pytest_wrapped.type == MongoBadDBError
def mongodb_database(uri, database_name, collection_name, stream_mode):
"""
Return MongoDB database
"""
database = MongoDB(uri,
database_name,
collection_name,
HWPCModel(),
stream_mode=stream_mode)
return database
def test_csvdb_bad_common(self):
"""
Test when file miss some common column
"""
csv_files = BAD_COMMON
while csv_files:
with pytest.raises(CsvBadCommonKeysError) as pytest_wrapped:
CsvDB(HWPCModel(), csv_files).connect()
assert pytest_wrapped.type == CsvBadCommonKeysError
csv_files = csv_files[1:]
async def test_read_one_json_object_received_from_the_socket(socket_db, unused_tcp_port):
json_reports = extract_json_report(1)
client = ClientThread(json_reports, unused_tcp_port)
client.start()
iterator = socket_db.iter(HWPCModel(), False)
report = await iterator.__anext__()
assert_report_equals(report, json_reports[0])
def test_crash_dispatcher(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result', HWPCModel())
pusher = PusherActor("pusher_mongodb",
PowerReport,
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: RAPLFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep', HWPCModel())
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
check_db()
def test_mongodb_save_basic_db(database):
"""
Test save mongodb collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb", "test_mongodb3")
mongodb.connect()
# Check if save work
basic_count = mongodb.collection.count_documents({})
for _ in range(2):
mongodb.save(gen_hwpc_report(), HWPCModel())
assert mongodb.collection.count_documents({}) == basic_count + 2
def puller(request, database, filt, stream_mode):
"""
Setup and Teardown for managing a PullerActor
setup: create a PullerActor and start its process
teardown: terminate the PullerActor process
"""
puller_actor = PullerActor("test_puller_" + str(request.node.name),
database,
filt,
HWPCModel(),
stream_mode=stream_mode,
level_logger=LOG_LEVEL)
yield puller_actor
def test_csvdb_second_primary_missing(self, csvdb):
"""
Create one full HWPCReport (the first), then return None
"""
csvdb.add_files(SECOND_PRIMARY_MISSING)
csvdb.connect()
group_name = [path.split('/')[-1][:-4] for path in SECOND_PRIMARY_MISSING]
csvdb_iter = csvdb.iter(HWPCModel(), False)
report = next(csvdb_iter)
for group in group_name:
assert group in report.groups
assert report.timestamp == timestamp_to_datetime(1539260664189)
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def test_csvdb_two_reports(self, csvdb):
"""
Create two full HWPCReport, then return None
"""
csvdb.add_files(BASIC_FILES)
csvdb.connect()
group_name = [path.split('/')[-1][:-4] for path in BASIC_FILES]
csvdb_iter = csvdb.iter(HWPCModel(), False)
for _ in range(2):
report = next(csvdb_iter)
for group in group_name:
assert group in report.groups
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def test_run_with_delay_between_message(unused_tcp_port, database, supervisor):
"""
run the same test but set a delay of 1s after sendig the 5th first messages
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL,
max_size=0)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_socket = SocketDB(unused_tcp_port)
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_socket",
input_socket,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL,
stream_mode=True)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
time.sleep(1)
json_reports = extract_json_report(10)
client = ClientThreadDelay(json_reports, unused_tcp_port)
client.start()
time.sleep(2)
check_db()
def test_csvdb_second_primary_missing(self):
"""
Create one full HWPCReport (the first), then return None
"""
csvdb = CsvDB(HWPCModel(), SECOND_PRIMARY_MISSING)
csvdb.connect()
group_name = [path.split('/')[-1] for path in SECOND_PRIMARY_MISSING]
csvdb_iter = iter(csvdb)
report = next(csvdb_iter)
for key in KEYS_COMMON:
assert key in report
for group in group_name:
assert group in report['groups']
assert report['timestamp'] == "1539260664189"
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def test_mongodb_read_basic_db(database):
"""
Test read mongodb collection
"""
# Load DB
mongodb = MongoDB(URI, "test_mongodb", "test_mongodb1")
# Check if we can reload after reading
mongodb.connect()
for _ in range(2):
mongodb_iter = mongodb.iter(HWPCModel(), False)
for _ in range(10):
assert next(mongodb_iter) is not None
with pytest.raises(StopIteration) as pytest_wrapped:
next(mongodb_iter)
assert pytest_wrapped.type == StopIteration
def test_csvdb_second_rapl_missing(self, csvdb):
"""
Create two reports, one is full, second without rapl, then return None
"""
csvdb.add_files(SECOND_RAPL_MISSING)
csvdb.connect()
group_name = [path.split('/')[-1][:-4] for path in SECOND_RAPL_MISSING]
csvdb_iter = csvdb.iter(HWPCModel(), False)
for i in range(2):
report = next(csvdb_iter)
for group in group_name:
if i == 1 and "rapl" in group:
assert group not in report.groups
else:
assert group in report.groups
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def puller(request, database, stream_mode):
"""
Setup and Teardown for managing a PullerActor
setup: create a PullerActor and start its process
teardown: terminate the PullerActor process
"""
dispatcher = DispatcherActor('dispatcher__', Mock(), Mock())
filt = Filter()
filt.filter(lambda msg: True, dispatcher)
puller_actor = PullerActor("test_puller_" + str(request.node.name),
database,
filt,
HWPCModel(),
stream_mode=stream_mode,
level_logger=LOG_LEVEL)
return puller_actor
def test_csvdb_two_reports(self):
"""
Create two full HWPCReport, then return None
"""
csvdb = CsvDB(HWPCModel(), BASIC_FILES)
csvdb.connect()
group_name = [path.split('/')[-1] for path in BASIC_FILES]
csvdb_iter = iter(csvdb)
for _ in range(2):
report = next(csvdb_iter)
for key in KEYS_COMMON:
assert key in report
for group in group_name:
assert group in report['groups']
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def test_csvdb_second_rapl_missing(self):
"""
Create two reports, one is full, second without rapl, then return None
"""
csvdb = CsvDB(HWPCModel(), SECOND_RAPL_MISSING)
csvdb.connect()
group_name = [path.split('/')[-1] for path in SECOND_RAPL_MISSING]
csvdb_iter = iter(csvdb)
for i in range(2):
report = next(csvdb_iter)
for key in KEYS_COMMON:
assert key in report
for group in group_name:
if i == 1 and "rapl" in group:
assert group not in report['groups']
else:
assert group in report['groups']
with pytest.raises(StopIteration) as pytest_wrapped:
next(csvdb_iter)
assert pytest_wrapped.type == StopIteration
def test_run(mongo_database, influx_database, supervisor):
# Pusher
output_db = InfluxDB(INFLUX_URI, INFLUX_PORT, INFLUX_DBNAME)
pusher = PusherActor("pusher_influx",
PowerModel(),
output_db,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
supervisor.join()
check_db(influx_database)
def test_run(database, supervisor):
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb", PowerModel(), output_mongodb, level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose:
DummyFormulaActor(name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(HWPCReport, HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher', formula_factory, route_table, level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb", input_mongodb, report_filter, HWPCModel(), level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current > number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
def __init__(self, component_group_name):
Generator.__init__(self, component_group_name)
self.model_factory = {
'HWPCReport': HWPCModel(),
'PowerReport': PowerModel(),
'FormulaReport': FormulaModel(),
'ControlReport': ControlModel(),
}
self.db_factory = {
'mongodb':
lambda db_config, _: MongoDB(db_config['uri'], db_config['db'],
db_config['collection']),
'socket':
lambda db_config, _: SocketDB(db_config['port']),
'csv':
lambda db_config, _: CsvDB(
current_path=os.getcwd()
if 'directory' not in db_config else db_config['directory'],
files=[] if 'files' not in db_config else db_config['files']),
'influxdb':
lambda db_config, _: InfluxDB(db_config['uri'], db_config['port'],
db_config['db']),
'opentsdb':
lambda db_config, _: OpenTSDB(db_config['uri'], db_config['port'],
db_config['metric_name']),
'prom':
lambda db_config, _: PrometheusDB(
db_config['port'], db_config['addr'], db_config['metric_name'],
db_config['metric_description'], self.model_factory[db_config[
'model']], db_config['aggregation_period']),
'tcp':
lambda db_config, group_name: IOTcpDB(db_config['uri'],
db_config['port'],
input=(group_name == "input")
)
}
def test_run(database, supervisor):
"""
Test the architecture when it has to deal with a lot of data, pulled with a
hight frequency. Input database contain a lot of data, that will overflow the
backend. Backend have to handle all of this data and continue to compute data
and writing them in real time.
Architecture :
- 1 puller (connected to MongoDB1 [collection test_hwrep], stream mode off)
- 1 dispatcher (HWPC dispatch rule (dispatch by SOCKET)
- 1 Dummy Formula
- 1 pusher (connected to MongoDB1 [collection test_result]
MongoDB1 content :
- 3000 HWPC repport with two socket and one RAPL_EVENT
Scenario:
- Launch the full architecture
Test if:
- each 50 ms, reports are writen in the output database
"""
# Pusher
output_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_result')
pusher = PusherActor("pusher_mongodb",
PowerModel(),
output_mongodb,
level_logger=LOG_LEVEL)
# Formula
formula_factory = (lambda name, verbose: DummyFormulaActor(
name, {'my_pusher': pusher}, level_logger=verbose))
# Dispatcher
route_table = RouteTable()
route_table.dispatch_rule(
HWPCReport,
HWPCDispatchRule(getattr(HWPCDepthLevel, 'SOCKET'), primary=True))
dispatcher = DispatcherActor('dispatcher',
formula_factory,
route_table,
level_logger=LOG_LEVEL)
# Puller
input_mongodb = MongoDB(DB_URI, 'MongoDB1', 'test_hwrep')
report_filter = Filter()
report_filter.filter(lambda msg: True, dispatcher)
puller = PullerActor("puller_mongodb",
input_mongodb,
report_filter,
HWPCModel(),
level_logger=LOG_LEVEL)
supervisor.launch_actor(pusher)
supervisor.launch_actor(dispatcher)
supervisor.launch_actor(puller)
t = time.time()
number_of_output_reports = 0
for i in range(3):
time.sleep(0.2)
current = get_number_of_output_reports()
assert current >= number_of_output_reports
number_of_output_reports = current
time.sleep(0.1)
supervisor.join()
