def run(self): ''' The main thread loop for this verifier. ''' # Create the shim self.logShim = DBShim(self.logServer, self.keyMgr) self.keyShim = DBShim(self.keyServer, self.keyMgr) # Run the crawler loop indefinitely... while self.running: logging.debug("Verifier " + str(self.id) + " is trying to grab a user session to verify.") (userId, sessionId) = self.selectRow() # Check to see if we found some valid data... if (userId != -1 and sessionId != -1): # Decrypt the user/session ID to get the original data userCT = userId.decode("hex") sessionCT = sessionId.decode("hex") key = hashlib.sha256(self.keyMgr.getMasterKey() + "log").digest() cipher = AES.new(key, AES.MODE_ECB) userPT = cipher.decrypt(userCT) sessionPT = cipher.decrypt(sessionCT) # Query the keys from the database logging.debug("Verifying: " + str(userPT) + " - " + str(sessionPT)) valueMap = {"userId" : userPT, "sessionId" : sessionPT} epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap, ["userId", "sessionId"]) key1 = epochKey[0]["key"] entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap, ["userId", "sessionId"]) key2 = entityKey[0]["key"] # Decrypt the keys using the 'verifier' policy logging.debug("Trying to decrypt") sk = self.encryptionModule.generateUserKey(['VERIFIER']) k1 = self.encryptionModule.decrypt(sk, key1)[1] # [1] to pull out plaintext, [0] is T/F flag k2 = self.encryptionModule.decrypt(sk, key2)[1] # [1] to pull out plaintext, [0] is T/F flag # Query the last digest from the database logging.debug("Decryption successful - continue with the verification process") entityDigest = self.logShim.executeMultiQuery("entity", valueMap, ["userId", "sessionId"]) digest = entityDigest[len(entityDigest) - 1]["digest"] # Query for the log now. valueMap = {"userId" : userId, "sessionId" : sessionId} logResult = self.logShim.executeMultiQuery("log", valueMap, []) log = {} userId = int(userPT) sessionId = int(sessionPT) log[(userId, sessionId)] = [] for i in range(0, len(logResult)): log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]]) # Verify the data extracted from the database... self.strongestVerify(userId, sessionId, log, k1, k2, digest, Logger.Logger.EPOCH_WINDOW_SIZE) # Don't hog the system resources time.sleep(15)
def run(self): ''' The main thread loop for this verifier. ''' # Create the shim self.logShim = DBShim(self.logServer) self.keyShim = DBShim(self.keyServer) # Run the crawler loop indefinitely... while self.running: print("Verifier " + str(self.id) + " is trying to grab a user session to verify...") (userId, sessionId) = self.selectRow() # Check to see if we found some valid data... if (userId != -1 and sessionId != -1): # Query the keys from the database valueMap = {"userId" : userId, "sessionId" : sessionId} epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap) key1 = epochKey[0]["key"] entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap) key2 = entityKey[0]["key"] # Decrypt the keys using the 'verifier' policy sk = self.encryptionModule.generateUserKey(['VERIFIER']) k1 = self.encryptionModule.decrypt(sk, key1) k2 = self.encryptionModule.decrypt(sk, key2) # QUIT PREMATURELY FOR TESTING PURPOSES sys.exit() # Query the last digest from the database entityDigest = self.logShim.executeMultiQuery("entity", valueMap) digest = entityDigest[len(entityDigest) - 1]["digest"] # Query for the log now. logResult = self.logShim.executeMultiQuery("log", valueMap) log = {} log[(userId, sessionId)] = [] for i in range(0, len(logResult)): log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]]) # Verify. print(log) self.strongestVerify(userId, sessionId, log, str(key1), str(key2), digest, Logger.Logger.EPOCH_WINDOW_SIZE) time.sleep(5)
def run(self): ''' The main loop for this client handler thread. Strip out a message, parse it according to the protocol, and then invoke the necessary commands. ''' global MSG_LOGIN self.shim = DBShim(self.params["AUDIT_USER_DB"], self.keyMgr) self.log = DBShim(self.params["LOG_DB"], self.keyMgr) while self.running: for client in self.clientList: message = client.sock.recv(self.server.BUFFSIZE) if message != None and message != "": try: parsedMsg = json.loads(message) if (int(parsedMsg['command']) == MSG_LOGIN): if (self.login(parsedMsg['parameters'])): client.sock.send('{"result":True,"message":"Login successful."}') elif (self.loggedIn == True): client.sock.send(self.parseMessage(parsedMsg)) except Exception as e: client.sock.send(str(e))
def run(self): logShim = DBShim(self.params["LOG_DB"], self.keyMgr) userShim = DBShim(self.params["USER_DB"], self.keyMgr) keyShim = DBShim(self.params["KEY_DB"], self.keyMgr) while(True): print("Audit task trying to verify the rule: " + str(self.rule.config)) # Parse the audit rule if (self.rule.role != None and self.rule.action != None): if (self.rule.object == None): valueMap = {"action" : self.rule.action} else: valueMap = {"action" : self.rule.action, "object" : self.rule.object} eventResults = logShim.executeMultiQuery("Event", valueMap, []) print(eventResults) for event in eventResults: userId = event["userId"] valueMap = {"roleId" : self.rule.role} userResults = userShim.executeMultiQuery("UserRole", valueMap, []) for user in userResults: if (int(user["userId"]) == int(userId)): print("Rule violation by user " + str(userId)) time.sleep(15)
class VerifyCrawler(threading.Thread): ''' This is an active thread that is responsible for serving all messages that come in from the keylogger. It simply strips them out of the socket and forwards them along to the logger actor via a message dictionary. ''' def __init__(self, vid, logServer, keyServer, keyMgr): ''' Constructor that stores the log server information. ''' threading.Thread.__init__(self) self.id = vid self.logServer = logServer self.keyServer = keyServer self.running = True # Build the encryption module self.keyMgr = keyMgr self.encryptionModule = EncryptionModule(keyMgr) # pass along the key manager reference # Generate the used entry bucket self.usedBin = {} self.MAX_TRIES = 10 # This can (and should) be configured by experimentation. # Configure the logger logFile = 'abls.log' logging.basicConfig(filename=logFile,level=logging.DEBUG) def run(self): ''' The main thread loop for this verifier. ''' # Create the shim self.logShim = DBShim(self.logServer, self.keyMgr) self.keyShim = DBShim(self.keyServer, self.keyMgr) # Run the crawler loop indefinitely... while self.running: logging.debug("Verifier " + str(self.id) + " is trying to grab a user session to verify.") (userId, sessionId) = self.selectRow() # Check to see if we found some valid data... if (userId != -1 and sessionId != -1): # Decrypt the user/session ID to get the original data userCT = userId.decode("hex") sessionCT = sessionId.decode("hex") key = hashlib.sha256(self.keyMgr.getMasterKey() + "log").digest() cipher = AES.new(key, AES.MODE_ECB) userPT = cipher.decrypt(userCT) sessionPT = cipher.decrypt(sessionCT) # Query the keys from the database logging.debug("Verifying: " + str(userPT) + " - " + str(sessionPT)) valueMap = {"userId" : userPT, "sessionId" : sessionPT} epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap, ["userId", "sessionId"]) key1 = epochKey[0]["key"] entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap, ["userId", "sessionId"]) key2 = entityKey[0]["key"] # Decrypt the keys using the 'verifier' policy logging.debug("Trying to decrypt") sk = self.encryptionModule.generateUserKey(['VERIFIER']) k1 = self.encryptionModule.decrypt(sk, key1)[1] # [1] to pull out plaintext, [0] is T/F flag k2 = self.encryptionModule.decrypt(sk, key2)[1] # [1] to pull out plaintext, [0] is T/F flag # Query the last digest from the database logging.debug("Decryption successful - continue with the verification process") entityDigest = self.logShim.executeMultiQuery("entity", valueMap, ["userId", "sessionId"]) digest = entityDigest[len(entityDigest) - 1]["digest"] # Query for the log now. valueMap = {"userId" : userId, "sessionId" : sessionId} logResult = self.logShim.executeMultiQuery("log", valueMap, []) log = {} userId = int(userPT) sessionId = int(sessionPT) log[(userId, sessionId)] = [] for i in range(0, len(logResult)): log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]]) # Verify the data extracted from the database... self.strongestVerify(userId, sessionId, log, k1, k2, digest, Logger.Logger.EPOCH_WINDOW_SIZE) # Don't hog the system resources time.sleep(15) def selectRow(self): ''' Randomly select a row from the database to check with strong verification. ''' userId = sessionId = 0 foundNewRow = False tries = 0 while not foundNewRow: result = self.logShim.randomQuery("log") if (len(result) > 0): userId = result[0]["userId"] sessionId = result[0]["sessionId"] if not ((userId, sessionId) in self.usedBin): self.usedBin[(userId, sessionId)] = 0 foundNewRow = True # Upgrade all the instances for for key in self.usedBin.keys(): self.usedBin[key] = self.usedBin[key] + 1 # See if we ran past the try cap tries = tries + 1 if (tries >= self.MAX_TRIES): tk1, tk2, maxNum = 0, 0, 0 for (k1, k2) in self.usedBin.keys(): if (self.usedBin[(k1, k2)] > maxNum): maxNum = self.usedBin[(k1, k2)] tk1 = -1 tk2 = -1 del self.usedBin[(tk1, tk2)] userId = tk1 sessionId = tk2 foundNewRow = True # we're going to retry a previous row else: userId = sessionId = -1 foundNewRow = True return (userId, sessionId) def strongestVerify(self, userId, sessionId, log, epochKey, entityKey, lastDigest, EPOCH_WINDOW_SIZE = Logger.Logger.EPOCH_WINDOW_SIZE): ''' Walks the log chain and epoch chain for verification, and computes the entity digests at every epoch cycle for comparison to check with the end result. Not publicly verifiable, and requires the initial epoch and entity keys. ''' ctChain = [] sha3 = Keccak.Keccak() # It is assumed that we would get this initial key from the trusted server... # This verification scheme is not possible without the epoch key... lastEpochDigest = hmac.new(epochKey, "0", hashlib.sha512).hexdigest() # Check to see if we even have anything to verify if not ((userId, sessionId) in log): return None else: # Handle the base of the chain first = log[(userId, sessionId)][0] firstPayload = str(userId) + str(sessionId) + str(0) + str(first[3]) + str(0) # Check the hash chain first xi = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex"))) computedV = sha3.Keccak((len(xi), xi)) assert(xi == first[4]) # Check the epoch chain next yi = hmac.new(epochKey, lastEpochDigest.encode("hex") + first[4].encode("hex"), hashlib.sha512).hexdigest() assert(yi == first[5]) # Compute the first part of the entity chain now lastEntityDigest = hmac.new(entityKey, xi, hashlib.sha512).hexdigest() entityKey = hmac.new(entityKey, "some constant value", hashlib.sha512).hexdigest() # Append the first message. ctChain.append(first[3]) # Walk the chain and make sure we can verify it... for i in range(1, len(log[(userId, sessionId)])): first = log[(userId, sessionId)][i] # Store the message firstMessage = first[3] # the message ctChain.append(firstMessage) # The other data... currentHash = first[4] # the hash previousHash = log[(userId, sessionId)][i - 1][4] # Verify that the first entry is correct firstPayload = str(userId) + str(0) + str(i) + str(firstMessage) + str(previousHash) firstComputedHash = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex"))) assert(currentHash == firstComputedHash) # Check the epoch chain to see if we need to cycle if ((i % EPOCH_WINDOW_SIZE) == 0): # Update the epoch key currKey = epochKey newKey = sha3.Keccak((len(bytes(currKey)), currKey.encode("hex"))) epochKey = newKey # Pull the last hash block length = len(log[(userId, sessionId)]) lastHash = log[(userId, sessionId)][i - 1][4] # Form the epoch block hash payload payload = str(lastEpochDigest) + str(lastHash) lastEpochDigest = hmac.new(newKey, payload, hashlib.sha512).hexdigest() # Compute the epoch chain value yi = hmac.new(epochKey, lastEpochDigest.encode("hex") + first[4].encode("hex"), hashlib.sha512).hexdigest() assert(yi == first[5]) # Compute the first part of the entity chain now lastEntityDigest = hmac.new(entityKey, first[4], hashlib.sha512).hexdigest() entityKey = hmac.new(entityKey, "some constant value", hashlib.sha512).hexdigest() assert(lastEntityDigest == lastDigest) logging.debug("Verification result:" + str(lastEntityDigest == lastDigest)) return ctChain def weakVerify(self, userId, sessionId, log, epochKey, entityKey, EPOCH_WINDOW_SIZE): ''' Only walks the log chain for verification. ''' ctChain = [] # Make sure we have something to verify first... if not ((userId, sessionId) in log): return None else: # Handle the base of the chain first = log[(userId, sessionId)][0] firstPayload = str(userId) + str(sessionId) + str(0) + str(first[3]) + str(0) digest = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex"))) assert(digest == first[4]) # Append the first message. ctChain.append(first[3]) # Walk the chain and make sure we can verify it... for i in range(1, len(log[(userId, sessionId)])): first = log[(userId, sessionId)][i] # Store the message firstMessage = first[3] # the message ctChain.append(firstMessage) # The other data... currentHash = first[4] # the hash previousHash = log[(userId, sessionId)][i - 1][4] # Verify that the first entry is correct firstPayload = str(userId) + str(0) + str(i) + str(firstMessage) + str(previousHash) firstComputedHash = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex"))) assert(currentHash == firstComputedHash) return ctChain
def bootstrap(): ''' Bootstrap the database from the database with some dummy data. ''' keyMgr = KeyManager(".") # Wipe the data if we're in debug mode print("Debug: Clearing the log database") # Check to see if we need to clear the table # This is specific to SQLite - coupling needs to be removed shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/log.db", keyMgr) tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='log'") if (len(tableResults) != 0): shim.executeRawQuery("DELETE FROM log") tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='entity'") if (len(tableResults) != 0): shim.executeRawQuery("DELETE FROM entity") tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='epoch'") if (len(tableResults) != 0): shim.executeRawQuery("DELETE FROM epoch") # Check to see if we need to clear the table # This is specific to SQLite - it needs to be less coupled to SQLite shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/users.db", keyMgr) tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='users'") if (len(tableResults) != 0): # Delete the contents in the table shim.executeRawQuery("DELETE FROM users") print("Initializing dummy data into the users table") # Create some dummy data for the database date = datetime.now() shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["alice", "*****@*****.**", "one", date, date], [False, False, False, False, False]) shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["bob", "*****@*****.**", "two", date, date], [False, False, False, False, False]) shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["chris", "*****@*****.**", "three", date, date], [False, False, False, False, False]) # Check to see if we need to clear the table # This is specific to SQLite - it needs to be less coupled to SQLite shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/audit_users.db", keyMgr) tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='audit_users'") if (len(tableResults) != 0): # Delete the contents in the table shim.executeRawQuery("DELETE FROM audit_users") print("Initializing dummy data into the users table") # Create some dummy data for the database date = datetime.now() alicePassword = "******" bobPassword = "******" chrisPassword = "******" # Generate the passwords/salts salt = str(uuid.uuid4()) hashed_password = hashlib.sha512(hashlib.sha512(alicePassword).hexdigest() + salt).hexdigest() shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["alice", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False]) salt = str(uuid.uuid4()) hashed_password = hashlib.sha512(hashlib.sha512(bobPassword).hexdigest() + salt).hexdigest() shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["bob", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False]) salt = str(uuid.uuid4()) hashed_password = hashlib.sha512(hashlib.sha512(chrisPassword).hexdigest() + salt).hexdigest() shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["chris", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False])
class AuditClientHandler(threading.Thread): ''' This is an active thread that is responsible for serving all messages that come in from audit proxy. ''' def __init__(self, serv, params, keyMgr): ''' Initialize the client handler with the parent server (AuditProxy) ''' threading.Thread.__init__(self) self.server = serv self.clientList = [] self.running = True # Setup the Python logger logFile = 'abls.log' logging.basicConfig(filename=logFile,level=logging.DEBUG) # Persist the key manager and params self.keyMgr = keyMgr self.params = params # Maintain login state of the user is logged in for this session self.loggedIn = False def run(self): ''' The main loop for this client handler thread. Strip out a message, parse it according to the protocol, and then invoke the necessary commands. ''' global MSG_LOGIN self.shim = DBShim(self.params["AUDIT_USER_DB"], self.keyMgr) self.log = DBShim(self.params["LOG_DB"], self.keyMgr) while self.running: for client in self.clientList: message = client.sock.recv(self.server.BUFFSIZE) if message != None and message != "": try: parsedMsg = json.loads(message) if (int(parsedMsg['command']) == MSG_LOGIN): if (self.login(parsedMsg['parameters'])): client.sock.send('{"result":True,"message":"Login successful."}') elif (self.loggedIn == True): client.sock.send(self.parseMessage(parsedMsg)) except Exception as e: client.sock.send(str(e)) def login(self, params): ''' Handle the user login process. ''' # Strip out the username and password userNameIndex = params.find(',') if (userNameIndex != -1): userName = params[0:userNameIndex] password = params[userNameIndex + 1:] # format: username,password # Fetch the salt from the database (there should only be one user by this name) record = self.shim.executeQuery("audit_users", "userName", userName, False) if (len(record) == 1): salt = record[0]["salt"] hashed_password = hashlib.sha512(password + salt).hexdigest() if (hashed_password == record[0]["password"]): self.loggedIn = True return True else: raise Exception("Invalid password for user: "******"Could not find the specified user: "******"client message: " + str(message)) # debug # Verify that the incoming message conforms to the protocol if (len(message) != 2): raise Exception("Invalid JSON string retrieved from client.") if not (('command' in message) and ('parameters' in message)): raise Exception("Invalid JSON string retrieved from client.") # Let it rip try: return self.execute(int(message['command']), message['parameters']) except: print("Error occured when executing the command") def execute(self, command, parameters): ''' Execute the command (if possible), or throw an exception if it's invalid. ''' # Bring the protocol event IDs into scope global MSG_SELECT_BY_USER global MSG_SELECT_BY_USER_SESSION result = "" # Handle the incoming message if (command == MSG_SELECT_BY_USER): #print("MSG_SELECT_BY_USER") #print(parameters[0]) valueMap = {"userId" : parameters[0]} rowMasks = ["userId"] try: results = self.log.executeMultiQuery("log", valueMap, rowMasks) # Format the results to return only the ciphertext logList = [] for i in range(0, len(results)): logList.append(results[i][4]) # these are encoded in "hex" - decode with .decode("hex") return json.dumps({"result" : True, "message" : json.dumps(logList)}) except Exception as e: print(e) elif (command == MSG_SELECT_BY_USER_SESSION): #print("MSG_SELECT_BY_USER_SESSION") #print(parameters[0]) valueMap = {"userId" : entry.userId, "sessionId" : entry.sessionId} rowMasks = ["userId", "sessionId"] try: results = self.log.executeMultiQuery("log", valueMap, rowMasks) # Format the results to return only the ciphertext logList = [] for i in range(0, len(results)): logList.append(results[i][4]) # these are encoded in "hex" - decode with .decode("hex") return json.dumps({"result" : True, "message" : json.dumps(logList)}) except: print(e) else: raise Exception("Unsupported audit command ID: " + str(command)) # Default return (if other messages don't work) return json.dumps({"result" : False, "message" : "Error parsing message: " + str(command)})