def insert_test():
print("Running insert tests")
conf = DatabaseManagerConfig()
conf.COLLECTION_NAME = "test_insert_collection"
manager = DatabaseManager(conf)
manager.collection.drop()
manager.id_map = {}
print(" Testing same key inserts")
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '3'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '100'}))
assert manager.get_all_keys() == ['1']
print(" Testing multiple key inserts")
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '2', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '3', 'sqn': '3'}))
_insert_check(manager, DatabaseEntry({'imsi': '4', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '5', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '6', 'sqn': '100'}))
assert set(manager.get_all_keys()) == set(['1', '2', '3', '4', '5', '6'])
manager.collection.drop()
print("Test Success")
def run_database_tests():
print("Running database performance tests")
num_nodes = 5
cm_list = []
tdm = TestDistributedManager()
for i in range(num_nodes):
print("Running database performance test")
cm_config = CentralManagerConfig()
sync_config = SyncManagerConfig()
dbm_config = DatabaseManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 5
sync_config.SYNC_REPORTED_MAX = 1000
sync_config.SYNC_ALL_INTERVAL = 1000
dbm_config.COLLECTION_NAME = "test_insert_" + str(i)
cm_config.OUTPUT_DIR = "./output/testing/performance_tests/baseline/node_" + str(
i)
cm_config.LOGGING_ENABLED = False
cm = CentralManager(cm_config)
cm.database_manager = DatabaseManager(dbm_config)
cm.database_manager.set_logger(cm._double_log)
cm.database_manager.collection.drop()
cm.distributed_manager = tdm
cm.sync_manager = SyncManager(sync_config)
cm.sync_manager.set_logger(cm._double_log)
cm.sync_manager.set_managers(cm.distributed_manager,
cm.database_manager)
cm_list.append(cm)
results = run_performance_tests(cm_list)
print("Results:", results)
for cm in cm_list:
cm.stop()
def simple_tests():
cm_config = CentralManagerConfig()
dbm_config = DatabaseManagerConfig()
dstm_config = DistributedManagerConfig()
sync_config = SyncManagerConfig()
sync_config.SYNC_INTERVAL = 15
cm_config.OUTPUT_DIR = "./output/testing/integrated_tests/simple"
cm_config.LOGGING_ENABLED = True
dbm_config.DATABASE_NAME = "test_db"
dbm_config.COLLECTION_NAME = "simple_test_collection"
dstm_config.BATCH_TIMEOUT = 0.5
dstm_config.BATCH_SIZE = 10
cm = CentralManager(cm_config)
cm.init_managers(dbm_config, dstm_config, sync_config, None)
cm.database_manager.collection.drop()
cm.start()
cm.stop()
def main():
# Set up config objects and parsing
cm_config = CentralManagerConfig()
nwm_config = None # NetworkManagerConfig()
dbm_config = DatabaseManagerConfig()
dstm_config = DistributedManagerConfig()
sync_config = SyncManagerConfig()
parse_args(cm_config=cm_config,
nwm_config=nwm_config,
dbm_config=dbm_config,
dstm_config=dstm_config)
# Create the central manager
cm = CentralManager(cm_config)
cm.init_managers(dbm_config, dstm_config, sync_config, nwm_config)
# Start running dAuth
cm.start() # Should block until ctrl-c
cm.stop()
from dAuth.managers import DatabaseManager
from dAuth.config import DatabaseManagerConfig
from dAuth.proto.database_entry import DatabaseEntry
conf = DatabaseManagerConfig()
conf.COLLECTION_NAME = "test_extract"
dbm = DatabaseManager(conf)
dbm.collection.drop()
dbm.start()
def printout(message):
from bson import json_util
print(json_util.dumps(message['o']))
dbm.trigger_handler.triggers.register_insert_trigger(
printout,
db_name=dbm.trigger_handler.db_name,
collection_name=dbm.trigger_handler.collection_name)
dbm.update_entry(
DatabaseEntry({
"imsi": "1234567890000",
"rand": "7144BFD6 9778F74D E505D589 E7B87878",
"sqn":
"1122334455667788990011223344556677889900112233445566778899001122",
"xres": "140F7A70 B7072208",
"kasme":
"40A88F3F C67D4111 F64FCBFF 15B08497 51E23254 0E6CF804 87658CC1 2EF4AC71",
def trigger_test():
print("Running trigger tests")
trigger_map = {}
def update_trigger_map(key):
if key not in trigger_map:
trigger_map[key] = 1
else:
trigger_map[key] += 1
conf = DatabaseManagerConfig()
conf.COLLECTION_NAME = "test_trigger_collection"
manager = DatabaseManager(conf)
manager.collection.drop()
manager.id_map = {}
manager.set_report_callback(update_trigger_map)
# Start triggers
manager.start()
try:
print(" Testing single trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '0'}))
time.sleep(1)
assert trigger_map == {'1': 1}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing multiple trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '1'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '2'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '3'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '4'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '5'}))
time.sleep(2)
assert trigger_map == {'1': 5}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing single trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '1'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '2'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '3'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '4'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '5'}))
time.sleep(2)
assert trigger_map == {'1': 1, '2': 1, '3': 1, '4': 1, '5': 1}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing multiple trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '01'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '02'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '03'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '04'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '05'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '11'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '12'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '13'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '14'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '15'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '21'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '22'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '23'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '24'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '25'}))
time.sleep(3)
assert trigger_map == {'1': 3, '2': 3, '3': 3, '4': 3, '5': 3}
manager.collection.drop()
manager.stop()
except AssertionError as e:
manager.stop()
print("Test Failed:", e)
return
print("Test Success")
def parse_args(cm_config: CentralManagerConfig = None,
nwm_config: NetworkManagerConfig = None,
dbm_config: DatabaseManagerConfig = None,
dstm_config: DistributedManagerConfig = None,
sync_config: SyncManagerConfig = None):
# Build parser and add arguments for each available config
parser = argparse.ArgumentParser(
description=
"dAuth arguments are prefixed by shorthand manager type, i.e. --cc- or --db-"
)
# Central manager
if cm_config:
parser.add_argument(
"--cc-id",
help="specify an id (randomly generated otherwise)",
default=cm_config.ID)
parser.add_argument("--cc-out",
help="directory for output, i.e. logs",
default=cm_config.OUTPUT_DIR)
# Network manager
if nwm_config:
parser.add_argument("--nwm-out",
help="directory for output, i.e. logs",
default=nwm_config.OUTPUT_DIR)
parser.add_argument("--nwm-port",
help="port to run gRPC server on",
default=nwm_config.GRPC_PORT)
parser.add_argument("--nwm-log-host",
help="host of the logging server",
default=nwm_config.LOGGING_SERVER_HOST)
parser.add_argument("--nwm-log-port",
help="port of the logging server",
default=nwm_config.LOGGING_SERVER_PORT)
# Database manager
if dbm_config:
parser.add_argument("--db-host",
help="host for the Mongo db",
default=dbm_config.HOST)
parser.add_argument("--db-port",
help="port for the Mongo db",
default=dbm_config.PORT)
parser.add_argument("--db-name",
help="name of the database",
default=dbm_config.DATABASE_NAME)
parser.add_argument("--db-collection",
help="name of collection within database",
default=dbm_config.COLLECTION_NAME)
# Distributed manager
if dstm_config:
parser.add_argument("--dst-validator-url",
help="url of validator to send transactions",
default=dstm_config.VALIDATOR_URL)
parser.add_argument("--dst-client-key-path",
help="local key path for the client",
default=dstm_config.CLIENT_KEY_PATH)
parser.add_argument("--dst-client-url",
help="client url of the sawtooth node",
default=dstm_config.CLIENT_URL)
parser.add_argument("--dst-batch-size",
help="max size of transaction batches",
default=dstm_config.BATCH_SIZE)
parser.add_argument("--dst-batch-timeout",
help="timeout before sending available batch",
default=dstm_config.BATCH_TIMEOUT)
parser.add_argument("--dst-batch-check-delay",
help="time between checking for new transactions",
default=dstm_config.BATCH_CHECK_DELAY)
args = parser.parse_args()
# Add results to each config
# Central Manager
if cm_config:
cm_config.ID = args.cc_id
cm_config.OUTPUT_DIR = args.cc_out
# Network manager
if nwm_config:
nwm_config.OUTPUT_DIR = args.nwm_out
nwm_config.GRPC_PORT = args.nwm_port
nwm_config.LOGGING_SERVER_HOST = args.nwm_log_host
nwm_config.LOGGING_SERVER_PORT = args.nwm_log_port
# Database manager
if dbm_config:
dbm_config.HOST = args.db_host
dbm_config.PORT = args.db_port
dbm_config.DATABASE_NAME = args.db_name
dbm_config.COLLECTION_NAME = args.db_collection
# Distributed manager
if dstm_config:
dstm_config.VALIDATOR_URL = args.dst_validator_url
dstm_config.CLIENT_KEY_PATH = args.dst_client_key_path
dstm_config.CLIENT_URL = args.dst_client_url
dstm_config.BATCH_SIZE = args.dst_batch_size
dstm_config.BATCH_TIMEOUT = args.dst_batch_timeout
dstm_config.BATCH_CHECK_DELAY = args.dst_batch_check_delay
def integrated_tests():
num_nodes = 5
nodes = []
for i in range(num_nodes):
cm_config = CentralManagerConfig()
dbm_config = DatabaseManagerConfig()
dstm_config = DistributedManagerConfig()
sync_config = SyncManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 1
sync_config.SYNC_REPORTED_MAX = 100
sync_config.SYNC_ALL_INTERVAL = 100
cm_config.OUTPUT_DIR = "./output/testing/integrated_tests/node_" + str(i)
cm_config.LOGGING_ENABLED = False
dbm_config.DATABASE_NAME = "test_db"
dbm_config.COLLECTION_NAME = "integrated_test_collection_" + str(i)
dstm_config.BATCH_TIMEOUT = 10
dstm_config.BATCH_SIZE = 100
dstm_config.VALIDATOR_URL = 'tcp://localhost:' + str(4004 + i)
dstm_config.CLIENT_URL = 'localhost:' + str(8008 + i)
cm = CentralManager(cm_config)
cm.init_managers(dbm_config, dstm_config, sync_config, None)
cm.database_manager.collection.drop()
nodes.append(cm)
for node in nodes:
threading.Thread(target=node.start, daemon=True).start()
print("Starting integration tests in 3 seconds")
time.sleep(3)
try:
num_inserts = 200
rate = 0.01
start = time.time()
check_time = time.time()
for i in range(num_inserts):
nodes[0].database_manager.update_entry(DatabaseEntry({"imsi": str(i), 'max_known_sqn': "1", "vectors": '[{"sqn":"1"}]'}))
time.sleep(rate)
if time.time() - check_time >= 1:
check_time = time.time()
time_passed = time.time() - start
print("Current state (during operations):")
for node in nodes:
count = node.database_manager.collection.count()
print(" total:", count, "rate: {0:.3f}".format(count/time_passed))
print()
print("Operations finished")
# Wait for propagations to finish
done = False
while not done:
done = True
time.sleep(2)
time_passed = time.time() - start
# check the current status of the other dbs
print("Current state")
for node in nodes:
count = node.database_manager.collection.estimated_document_count()
done &= count >= num_inserts
print(" total:", count, "rate: {0:.3f}".format(count/time_passed))
print()
print("Test finished successfully")
except KeyboardInterrupt:
print("Skipping test...")
print("Stopping and cleaning up")
for node in nodes:
node.stop()
node.database_manager.collection.drop()
time.sleep(2)
print("Done")
exit(0)