def apply(self, transaction, context):
self.log("Apply called")
# unpack transaction info
action, key, data = unpack_transaction(transaction)
# get the current state
state = get_state_data(key, context)
if action == 'set':
self.log(" Action is 'set'")
# use protobuf data to create a DatabaseEntry and execute it
entry = DatabaseEntry(data)
try:
self.apply_callback(entry.key())
except ValueError:
self.log(" Database entry apply failed")
# update the state
updated_state = dict(state.items())
updated_state[key] = data
set_state_data(key, updated_state, context)
else:
raise InternalError('Invalid function requested to be executed by CCellular Handler')
def periodic_insert(manager:DistributedManager, iterations):
time.sleep(1)
print("Running insert tests")
try:
print(" Testing inserts and reads")
for i in range(iterations):
manager.update_entry(DatabaseEntry({"imsi": str(i), "sqn": str(i) }))
time.sleep(2)
for i in range(iterations):
entry = manager.get_entry(str(i))
assert entry != None
assert entry.key() == str(i) and entry.to_dict()['sqn'] == str(i)
print(" Testing updates and reads")
for i in range(iterations):
manager.update_entry(DatabaseEntry({"imsi": str(i), "sqn": str(i+10) }))
time.sleep(2)
for i in range(iterations):
entry = manager.get_entry(str(i))
assert entry != None
assert entry.key() == str(i) and entry.to_dict()['sqn'] == str(i+10)
assert manager.get_all_keys() == set([str(i) for i in range(iterations)])
print("Test Success")
except AssertionError as e:
print("Failed Test:", e)
print("(^c to end)")
def build_payload(action, entry: DatabaseEntry):
payload = {
"verb": action,
"key": entry.key(),
"data": entry.get_serialized_message()
}
return cbor.dumps(payload)
def build_database_entry(imsi:int) -> DatabaseEntry:
if use_old_format:
return DatabaseEntry({
"imsi": "1234567890" + str(imsi),
"rand" : hex_obj,
"sqn" : "1"
})
else:
return DatabaseEntry({
"imsi": "1234567890" + str(imsi),
"rand" : "7144BFD6 9778F74D E505D589 E7B87878",
"sqn" : "1122334455667788990011223344556677889900112233445566778899001122",
"xres" : "140F7A70 B7072208",
"kasme" : "40A88F3F C67D4111 F64FCBFF 15B08497 51E23254 0E6CF804 87658CC1 2EF4AC71",
"autn" : "90067853 A7608000 0DD7E6E1 6C269848",
"ck" : "05ABC825 BECE744D 776425AF D8141C43",
"ak" : "90067853 E8E2",
"ik" : "EAE625B0 15306BB8 243398FC AB92B660"
})
def test_overwrite_max_full():
conf = SyncManagerConfig()
manager = SyncManager(conf)
dist = TestDistributedManager()
db = TestDatabaseManager()
manager.set_managers(dist, db)
# manager.set_logger(print)
db.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
db.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
db.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "1",
"vectors": '[{"sqn":"1"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
dist.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "2",
"vectors": '[{"sqn":"2"}]'
}))
assert db.get_all_keys() == {"1", "2", "3"}
assert dist.get_all_keys() == {"1", "2", "3"}
for imsi in db.get_all_keys():
assert _check_entry(db.get_entry(imsi), imsi, "1", [{'sqn': '1'}])
assert _check_entry(dist.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
manager.report_all()
manager.sync_reported()
for imsi in db.get_all_keys():
assert _check_entry(db.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
assert _check_entry(dist.get_entry(imsi), imsi, "2", [{'sqn': '2'}])
def get(self, key):
self.log("Get entry called with key: " + str(key))
address = make_ccellular_address(key)
result = self._send_request("state/{}".format(address), name=key)
if result != None:
try:
json_result = json.loads(result)
data_response = json_result['data']
b64data = yaml.safe_load(data_response)
b64decoded = base64.b64decode(b64data)
cbor_decoded = cbor.loads(b64decoded)
return DatabaseEntry(cbor_decoded[key])
except BaseException as e:
print("Received a base exception:", e)
return None
def main(args_in=None):
op_choices = ['i', 'd']
parser = argparse.ArgumentParser(
description="Small utility to perform a database operation on a MongoDB."
" Optional args will use insert and config defaults")
parser.add_argument("--op-type",
default="i",
choices=op_choices,
help="operation type to perform")
parser.add_argument("--db-host", default="localhost", help="MongoDB host")
parser.add_argument("--db-port",
default=DatabaseManagerConfig.PORT,
type=int,
help="MongoDB port")
parser.add_argument("--db-name",
default=DatabaseManagerConfig.DATABASE_NAME,
help="Database name to get collection from")
parser.add_argument("--db-coll",
default=DatabaseManagerConfig.COLLECTION_NAME,
help="Collection name to operate on")
parser.add_argument(
"data",
type=json.loads,
help="Data to use in the operation in the form of a dictionary")
args = parser.parse_args(args_in)
if type(args.data) is not dict:
raise ValueError("Data must be in the form of dictionary")
# Create a client for the db and get the collection
client = MongoClient(args.db_host, args.db_port)
db = client[args.db_name]
collection = db[args.db_coll]
data = args.data
entry = DatabaseEntry(args.data)
if args.op_type == 'i':
do_insert(collection, entry)
elif args.op_type == 'd':
do_delete(collection, entry)
def insert(collection: Collection, entry: DatabaseEntry):
return collection.update_one(entry.get_filter(), {
"$set": entry.get_data()
},
upsert=True).upserted_id
def update_entry_and_report(self, entry: DatabaseEntry):
self.update_entry(entry)
self.report_update(entry.key())
dbm.start()
def printout(message):
from bson import json_util
print(json_util.dumps(message['o']))
dbm.trigger_handler.triggers.register_insert_trigger(
printout,
db_name=dbm.trigger_handler.db_name,
collection_name=dbm.trigger_handler.collection_name)
dbm.update_entry(
DatabaseEntry({
"imsi": "1234567890000",
"rand": "7144BFD6 9778F74D E505D589 E7B87878",
"sqn":
"1122334455667788990011223344556677889900112233445566778899001122",
"xres": "140F7A70 B7072208",
"kasme":
"40A88F3F C67D4111 F64FCBFF 15B08497 51E23254 0E6CF804 87658CC1 2EF4AC71",
"autn": "90067853 A7608000 0DD7E6E1 6C269848",
"ck": "05ABC825 BECE744D 776425AF D8141C43",
"ak": "90067853 E8E2",
"ik": "EAE625B0 15306BB8 243398FC AB92B660"
}))
import time
time.sleep(1)
dbm.stop()
def insert_test():
print("Running insert tests")
conf = DatabaseManagerConfig()
conf.COLLECTION_NAME = "test_insert_collection"
manager = DatabaseManager(conf)
manager.collection.drop()
manager.id_map = {}
print(" Testing same key inserts")
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '3'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '100'}))
assert manager.get_all_keys() == ['1']
print(" Testing multiple key inserts")
_insert_check(manager, DatabaseEntry({'imsi': '1', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '2', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '3', 'sqn': '3'}))
_insert_check(manager, DatabaseEntry({'imsi': '4', 'sqn': '2'}))
_insert_check(manager, DatabaseEntry({'imsi': '5', 'sqn': '1'}))
_insert_check(manager, DatabaseEntry({'imsi': '6', 'sqn': '100'}))
assert set(manager.get_all_keys()) == set(['1', '2', '3', '4', '5', '6'])
manager.collection.drop()
print("Test Success")
def update_entry(self, entry: DatabaseEntry):
self.db[entry.key()] = entry.get_serialized_message()
def update_entry(self, entry: DatabaseEntry):
self.log("Updating entry: " + str(entry.to_dict()))
self.client.set_entry(entry)
def _needs_update(self, entry:DatabaseEntry, compare_to:DatabaseEntry):
if entry == None:
if compare_to == None:
self.log("Both entries are None")
else:
self.log("Entry is None")
# Anything is better than nothing
return compare_to != None
# The logic for needing an update:
# - If the max known is lower
# - Else if max known is equal:
# - If max current is lower
# - Else if max current is equal and length of vectors is *longer*
if entry != None and compare_to != None:
self.log("Entry compare (max, cur): entry ({}, {}), compare_to ({}, {})"\
.format(entry.get_max_known_sqn(),
entry.get_max_current_sqn(),
compare_to.get_max_known_sqn(),
compare_to.get_max_current_sqn()))
if entry.get_max_known_sqn() < compare_to.get_max_known_sqn():
self.log("Entry has lower max sqn")
return True
elif entry.get_max_known_sqn() == compare_to.get_max_known_sqn():
if entry.get_max_current_sqn() == compare_to.get_max_current_sqn():
# A list with LESS entries is more up to date
self.log("Entry current max equal")
return len(entry.get_vectors()) > len(compare_to.get_vectors())
else:
return entry.get_max_current_sqn() < compare_to.get_max_current_sqn()
else:
self.log("Entry has a higher max seqnum")
return False
self.log("Compare To entry is None")
return False
def get_entry(self, key):
self.log("Getting entry with key: " + str(key))
data = self.collection.find_one({"imsi": key})
if data != None:
return DatabaseEntry(data)
return None
def trigger_test():
print("Running trigger tests")
trigger_map = {}
def update_trigger_map(key):
if key not in trigger_map:
trigger_map[key] = 1
else:
trigger_map[key] += 1
conf = DatabaseManagerConfig()
conf.COLLECTION_NAME = "test_trigger_collection"
manager = DatabaseManager(conf)
manager.collection.drop()
manager.id_map = {}
manager.set_report_callback(update_trigger_map)
# Start triggers
manager.start()
try:
print(" Testing single trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '0'}))
time.sleep(1)
assert trigger_map == {'1': 1}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing multiple trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '1'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '2'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '3'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '4'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '5'}))
time.sleep(2)
assert trigger_map == {'1': 5}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing single trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '1'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '2'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '3'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '4'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '5'}))
time.sleep(2)
assert trigger_map == {'1': 1, '2': 1, '3': 1, '4': 1, '5': 1}
trigger_map.clear()
manager.collection.drop()
manager.id_map = {}
print(" Testing multiple trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '01'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '02'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '03'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '04'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '05'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '11'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '12'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '13'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '14'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '15'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn': '21'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn': '22'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn': '23'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn': '24'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn': '25'}))
time.sleep(3)
assert trigger_map == {'1': 3, '2': 3, '3': 3, '4': 3, '5': 3}
manager.collection.drop()
manager.stop()
except AssertionError as e:
manager.stop()
print("Test Failed:", e)
return
print("Test Success")
def update_entry(self, entry: DatabaseEntry):
self.log("Updating entry: " + str(entry.to_dict()))
result_id = MongoDBOperations.insert(self.collection, entry)
if result_id != None:
self.id_map[result_id] = entry.key()
def _check_entry(entry: DatabaseEntry, imsi, max_known_sqn, vectors):
return entry.key() == imsi and\
int(entry.get_max_known_sqn()) == int(max_known_sqn) and\
entry.get_vectors() == vectors
def test_simple_sync():
conf = SyncManagerConfig()
manager = SyncManager(conf)
dist = TestDistributedManager()
db = TestDatabaseManager()
manager.set_managers(dist, db)
# manager.set_logger(print)
db.update_entry(
DatabaseEntry({
"imsi": '2',
'max_known_sqn': "1",
"vectors": "[]"
}))
dist.update_entry(
DatabaseEntry({
"imsi": '1',
'max_known_sqn': "1",
"vectors": "[]"
}))
assert db.get_all_keys() == {"2"}
assert dist.get_all_keys() == {"1"}
manager.report_all() # sets all keys to reported
manager.sync_reported() # does the actual syncing
assert db.get_all_keys() == {"1", "2"}
assert dist.get_all_keys() == {"1", "2"}
db.update_entry(
DatabaseEntry({
"imsi": '3',
'max_known_sqn': "1",
"vectors": "[]"
}))
db.update_entry(
DatabaseEntry({
"imsi": '4',
'max_known_sqn': "1",
"vectors": "[]"
}))
db.update_entry(
DatabaseEntry({
"imsi": '5',
'max_known_sqn': "1",
"vectors": "[]"
}))
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2"}
manager.report_update("4")
manager.sync_reported()
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2", "4"}
manager.report_all()
manager.sync_reported()
assert db.get_all_keys() == {"1", "2", "3", "4", "5"}
assert dist.get_all_keys() == {"1", "2", "3", "4", "5"}
def trigger_test():
print("Running trigger tests")
trigger_map = {}
def update_trigger_map(key):
if key not in trigger_map:
trigger_map[key] = 1
conf = DistributedManagerConfig()
conf.BATCH_TIMEOUT = 0.5
conf.BATCH_SIZE = 20
manager = DistributedManager(conf)
manager.set_report_callback(update_trigger_map)
# Start triggers
manager.start()
threading.Thread(target=manager.run_main, daemon=True).start()
try:
print(" Testing single trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'0'}))
time.sleep(2)
assert trigger_map == {'1': 1}
trigger_map.clear()
print(" Testing multiple trigger single key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'1'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'2'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'3'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'4'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'5'}))
time.sleep(2)
print(trigger_map)
assert trigger_map == {'1': 1}
trigger_map.clear()
print(" Testing single trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'1'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn':'2'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn':'3'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn':'4'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn':'5'}))
time.sleep(2)
assert trigger_map == {'1': 1, '2': 1, '3': 1, '4': 1, '5': 1}
trigger_map.clear()
print(" Testing multiple trigger multiple key")
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'01'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn':'02'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn':'03'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn':'04'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn':'05'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'11'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn':'12'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn':'13'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn':'14'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn':'15'}))
manager.update_entry(DatabaseEntry({'imsi': '1', 'sqn':'21'}))
manager.update_entry(DatabaseEntry({'imsi': '2', 'sqn':'22'}))
manager.update_entry(DatabaseEntry({'imsi': '3', 'sqn':'23'}))
manager.update_entry(DatabaseEntry({'imsi': '4', 'sqn':'24'}))
manager.update_entry(DatabaseEntry({'imsi': '5', 'sqn':'25'}))
time.sleep(2)
assert trigger_map == {'1': 1, '2': 1, '3': 1, '4': 1, '5': 1}
manager.stop()
except AssertionError as e:
manager.stop()
print("Test Failed:", e)
return
print("Test Success")
def update(collection: Collection, entry: DatabaseEntry):
collection.update_one(entry.get_filter(), entry.get_data())
def integrated_tests():
num_nodes = 5
nodes = []
for i in range(num_nodes):
cm_config = CentralManagerConfig()
dbm_config = DatabaseManagerConfig()
dstm_config = DistributedManagerConfig()
sync_config = SyncManagerConfig()
sync_config.SYNC_REPORTED_INTERVAL = 1
sync_config.SYNC_REPORTED_MAX = 100
sync_config.SYNC_ALL_INTERVAL = 100
cm_config.OUTPUT_DIR = "./output/testing/integrated_tests/node_" + str(i)
cm_config.LOGGING_ENABLED = False
dbm_config.DATABASE_NAME = "test_db"
dbm_config.COLLECTION_NAME = "integrated_test_collection_" + str(i)
dstm_config.BATCH_TIMEOUT = 10
dstm_config.BATCH_SIZE = 100
dstm_config.VALIDATOR_URL = 'tcp://localhost:' + str(4004 + i)
dstm_config.CLIENT_URL = 'localhost:' + str(8008 + i)
cm = CentralManager(cm_config)
cm.init_managers(dbm_config, dstm_config, sync_config, None)
cm.database_manager.collection.drop()
nodes.append(cm)
for node in nodes:
threading.Thread(target=node.start, daemon=True).start()
print("Starting integration tests in 3 seconds")
time.sleep(3)
try:
num_inserts = 200
rate = 0.01
start = time.time()
check_time = time.time()
for i in range(num_inserts):
nodes[0].database_manager.update_entry(DatabaseEntry({"imsi": str(i), 'max_known_sqn': "1", "vectors": '[{"sqn":"1"}]'}))
time.sleep(rate)
if time.time() - check_time >= 1:
check_time = time.time()
time_passed = time.time() - start
print("Current state (during operations):")
for node in nodes:
count = node.database_manager.collection.count()
print(" total:", count, "rate: {0:.3f}".format(count/time_passed))
print()
print("Operations finished")
# Wait for propagations to finish
done = False
while not done:
done = True
time.sleep(2)
time_passed = time.time() - start
# check the current status of the other dbs
print("Current state")
for node in nodes:
count = node.database_manager.collection.estimated_document_count()
done &= count >= num_inserts
print(" total:", count, "rate: {0:.3f}".format(count/time_passed))
print()
print("Test finished successfully")
except KeyboardInterrupt:
print("Skipping test...")
print("Stopping and cleaning up")
for node in nodes:
node.stop()
node.database_manager.collection.drop()
time.sleep(2)
print("Done")
exit(0)
def get_entry(self, key):
if key in self.db:
return DatabaseEntry(self.db.get(key))
return None
def delete(collection: Collection, entry: DatabaseEntry):
collection.delete_one(entry.get_filter())
def _insert_check(manager: DatabaseManager, entry: DatabaseEntry):
manager.update_entry(entry)
assert manager.get_entry(entry.key()).to_dict() == entry.to_dict()