def init_managers(self, dbm_conf, dist_conf, sync_conf, nwm_config):
self.database_manager = DatabaseManager(dbm_conf)
self.distributed_manager = DistributedManager(dist_conf)
self.sync_manager = SyncManager(sync_conf)
self.database_manager.set_logger(self._double_log)
self.distributed_manager.set_logger(self._double_log)
self.sync_manager.set_logger(self._double_log)
self.sync_manager.set_managers(self.distributed_manager, self.database_manager)
def test_overwrite_max_full():
conf = SyncManagerConfig()
manager = SyncManager(conf)
dist = TestDistributedManager()
db = TestDatabaseManager()
manager.set_managers(dist, db)
# manager.set_logger(print)
db.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
db.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
db.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
assert db.get_all_keys() == {"1", "2", "3"}
assert dist.get_all_keys() == {"1", "2", "3"}
for imsi in db.get_all_keys():
assert _check_entry(db.get_entry(imsi), imsi, "1", [{'sqn': '1'}])
assert _check_entry(dist.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
manager.report_all()
manager.sync_reported()
for imsi in db.get_all_keys():
assert _check_entry(db.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
assert _check_entry(dist.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
def run_distributed_tests():
print("Running distributed performance tests")
num_nodes = 5
cm_list = []
for i in range(num_nodes):
cm_config = CentralManagerConfig()
sync_config = SyncManagerConfig()
dstm_config = DistributedManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 2.5
sync_config.SYNC_REPORTED_MAX = 100
sync_config.SYNC_ALL_INTERVAL = 1000
dstm_config.BATCH_TIMEOUT = 5
dstm_config.BATCH_SIZE = 20
dstm_config.VALIDATOR_URL = 'tcp://localhost:' + str(4004 + i)
dstm_config.CLIENT_URL = 'localhost:' + str(8008 + i)
cm_config.OUTPUT_DIR = "./output/testing/performance_tests/baseline/node_" + str(i)
cm_config.LOGGING_ENABLED = False
cm = CentralManager(cm_config)
cm.database_manager = TestDatabaseManager()
cm.distributed_manager = DistributedManager(dstm_config)
cm.distributed_manager.set_logger(cm._double_log)
cm.sync_manager = SyncManager(sync_config)
cm.sync_manager.set_logger(cm._double_log)
cm.sync_manager.set_managers(cm.distributed_manager, cm.database_manager)
cm_list.append(cm)
results = run_performance_tests(cm_list)
print("Results:", results)
def run_baseline_tests():
print("Running baseline performance tests")
num_nodes = 5
cm_list = []
tdm = TestDistributedManager()
for i in range(num_nodes):
cm_config = CentralManagerConfig()
sync_config = SyncManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 0.5
sync_config.SYNC_REPORTED_MAX = 1000
sync_config.SYNC_ALL_INTERVAL = 1000
cm_config.OUTPUT_DIR = "./output/testing/performance_tests/baseline/node_" + str(i)
cm_config.LOGGING_ENABLED = False
cm = CentralManager(cm_config)
cm.database_manager = TestDatabaseManager()
cm.distributed_manager = tdm
cm.sync_manager = SyncManager(sync_config)
cm.sync_manager.set_logger(cm._double_log)
cm.sync_manager.set_managers(cm.distributed_manager, cm.database_manager)
cm_list.append(cm)
results = run_performance_tests(cm_list)
print("Results:", results)
def run_database_tests():
print("Running database performance tests")
num_nodes = 5
cm_list = []
tdm = TestDistributedManager()
for i in range(num_nodes):
print("Running database performance test")
cm_config = CentralManagerConfig()
sync_config = SyncManagerConfig()
dbm_config = DatabaseManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 5
sync_config.SYNC_REPORTED_MAX = 1000
sync_config.SYNC_ALL_INTERVAL = 1000
dbm_config.COLLECTION_NAME = "test_insert_" + str(i)
cm_config.OUTPUT_DIR = "./output/testing/performance_tests/baseline/node_" + str(
i)
cm_config.LOGGING_ENABLED = False
cm = CentralManager(cm_config)
cm.database_manager = DatabaseManager(dbm_config)
cm.database_manager.set_logger(cm._double_log)
cm.database_manager.collection.drop()
cm.distributed_manager = tdm
cm.sync_manager = SyncManager(sync_config)
cm.sync_manager.set_logger(cm._double_log)
cm.sync_manager.set_managers(cm.distributed_manager,
cm.database_manager)
cm_list.append(cm)
results = run_performance_tests(cm_list)
print("Results:", results)
for cm in cm_list:
cm.stop()
class CentralManager:
def __init__(self, conf:CentralManagerConfig):
self.conf = conf
self.id = conf.ID or random_string() # generate random id if one is not configured
self.database_manager = None
self.distributed_manager = None
self.sync_manager = None
# logging
if conf.LOGGING_ENABLED:
logger = logging.getLogger("central_manager-" + self.id)
logger.setLevel(logging.DEBUG)
os.makedirs(conf.OUTPUT_DIR, exist_ok=True)
fh = logging.FileHandler(os.path.join(conf.OUTPUT_DIR, "central_manager.log"))
fh.setLevel(logging.DEBUG)
logger.addHandler(fh)
self.local_logger = logger.debug
else:
self.local_logger = None
self.remote_logger = None
# internal
self._running = False
self.log("Central Manager created with ID: " + self.id)
# Creates the managers with the corresponding configs
# Must be called before start
def init_managers(self, dbm_conf, dist_conf, sync_conf, nwm_config):
self.database_manager = DatabaseManager(dbm_conf)
self.distributed_manager = DistributedManager(dist_conf)
self.sync_manager = SyncManager(sync_conf)
self.database_manager.set_logger(self._double_log)
self.distributed_manager.set_logger(self._double_log)
self.sync_manager.set_logger(self._double_log)
self.sync_manager.set_managers(self.distributed_manager, self.database_manager)
# Starts all managers
def start(self):
self.log("Start called")
if not self._running:
self._running = True
self.log("Starting all managers")
try:
self.database_manager.start()
self.distributed_manager.start()
self.sync_manager.start()
# self.network_manager.start()
except Exception as e:
self.log(" Failed to start managers: " + str(e))
# Run the distributed manager's client
self.distributed_manager.run_main()
else:
self.log(" Already running")
# Stops all managers
def stop(self):
self.log("Stop called")
if self._running:
self._running = True
self.log("Stopping all managers")
try:
self.database_manager.stop()
self.distributed_manager.stop()
self.sync_manager.stop()
# self.network_manager.stop()
except Exception as e:
self.log(" Failed to stop managers: " + str(e))
else:
self.log(" Not running")
# Log the message
def log(self, message):
if self.id:
self._double_log("Central Manager:"+self.id, message)
else:
self._double_log("Central Manager", message)
# Sends message to local and/or remote logger
def _double_log(self, category, message):
if self.conf.LOGGING_ENABLED:
if self.local_logger:
self.local_logger("<{0}> {1}".format(category, message))
if self.remote_logger:
self.remote_logger(category, message)
# Debug logging function, simply prints
@staticmethod
def _debug_log(category, message):
print("<{0}> {1}".format(category, message))
def test_simple_sync():
conf = SyncManagerConfig()
manager = SyncManager(conf)
dist = TestDistributedManager()
db = TestDatabaseManager()
manager.set_managers(dist, db)
# manager.set_logger(print)
db.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "1",
"vectors": "[]"
}))
dist.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "1",
"vectors": "[]"
}))
assert db.get_all_keys() == {"2"}
assert dist.get_all_keys() == {"1"}
manager.report_all() # sets all keys to reported
manager.sync_reported() # does the actual syncing
assert db.get_all_keys() == {"1", "2"}
assert dist.get_all_keys() == {"1", "2"}
db.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "1",
"vectors": "[]"
}))
db.update_entry(
DatabaseEntry({
"imsi": '4',
'max_known_sqn': "1",
"vectors": "[]"
}))
db.update_entry(
DatabaseEntry({
"imsi": '5',
'max_known_sqn': "1",
"vectors": "[]"
}))
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2"}
manager.report_update("4")
manager.sync_reported()
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2", "4"}
manager.report_all()
manager.sync_reported()
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2", "3", "4", "5"}