def run(self):
''' The main thread loop for this verifier.
'''
# Create the shim
self.logShim = DBShim(self.logServer, self.keyMgr)
self.keyShim = DBShim(self.keyServer, self.keyMgr)
# Run the crawler loop indefinitely...
while self.running:
logging.debug("Verifier " + str(self.id) + " is trying to grab a user session to verify.")
(userId, sessionId) = self.selectRow()
# Check to see if we found some valid data...
if (userId != -1 and sessionId != -1):
# Decrypt the user/session ID to get the original data
userCT = userId.decode("hex")
sessionCT = sessionId.decode("hex")
key = hashlib.sha256(self.keyMgr.getMasterKey() + "log").digest()
cipher = AES.new(key, AES.MODE_ECB)
userPT = cipher.decrypt(userCT)
sessionPT = cipher.decrypt(sessionCT)
# Query the keys from the database
logging.debug("Verifying: " + str(userPT) + " - " + str(sessionPT))
valueMap = {"userId" : userPT, "sessionId" : sessionPT}
epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap, ["userId", "sessionId"])
key1 = epochKey[0]["key"]
entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap, ["userId", "sessionId"])
key2 = entityKey[0]["key"]
# Decrypt the keys using the 'verifier' policy
logging.debug("Trying to decrypt")
sk = self.encryptionModule.generateUserKey(['VERIFIER'])
k1 = self.encryptionModule.decrypt(sk, key1)[1] # [1] to pull out plaintext, [0] is T/F flag
k2 = self.encryptionModule.decrypt(sk, key2)[1] # [1] to pull out plaintext, [0] is T/F flag
# Query the last digest from the database
logging.debug("Decryption successful - continue with the verification process")
entityDigest = self.logShim.executeMultiQuery("entity", valueMap, ["userId", "sessionId"])
digest = entityDigest[len(entityDigest) - 1]["digest"]
# Query for the log now.
valueMap = {"userId" : userId, "sessionId" : sessionId}
logResult = self.logShim.executeMultiQuery("log", valueMap, [])
log = {}
userId = int(userPT)
sessionId = int(sessionPT)
log[(userId, sessionId)] = []
for i in range(0, len(logResult)):
log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]])
# Verify the data extracted from the database...
self.strongestVerify(userId, sessionId, log, k1, k2, digest, Logger.Logger.EPOCH_WINDOW_SIZE)
# Don't hog the system resources
time.sleep(15)
def run(self):
''' The main thread loop for this verifier.
'''
# Create the shim
self.logShim = DBShim(self.logServer)
self.keyShim = DBShim(self.keyServer)
# Run the crawler loop indefinitely...
while self.running:
print("Verifier " + str(self.id) + " is trying to grab a user session to verify...")
(userId, sessionId) = self.selectRow()
# Check to see if we found some valid data...
if (userId != -1 and sessionId != -1):
# Query the keys from the database
valueMap = {"userId" : userId, "sessionId" : sessionId}
epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap)
key1 = epochKey[0]["key"]
entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap)
key2 = entityKey[0]["key"]
# Decrypt the keys using the 'verifier' policy
sk = self.encryptionModule.generateUserKey(['VERIFIER'])
k1 = self.encryptionModule.decrypt(sk, key1)
k2 = self.encryptionModule.decrypt(sk, key2)
# QUIT PREMATURELY FOR TESTING PURPOSES
sys.exit()
# Query the last digest from the database
entityDigest = self.logShim.executeMultiQuery("entity", valueMap)
digest = entityDigest[len(entityDigest) - 1]["digest"]
# Query for the log now.
logResult = self.logShim.executeMultiQuery("log", valueMap)
log = {}
log[(userId, sessionId)] = []
for i in range(0, len(logResult)):
log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]])
# Verify.
print(log)
self.strongestVerify(userId, sessionId, log, str(key1), str(key2), digest, Logger.Logger.EPOCH_WINDOW_SIZE)
time.sleep(5)
def run(self):
''' The main loop for this client handler thread. Strip out a message,
parse it according to the protocol, and then invoke the necessary commands.
'''
global MSG_LOGIN
self.shim = DBShim(self.params["AUDIT_USER_DB"], self.keyMgr)
self.log = DBShim(self.params["LOG_DB"], self.keyMgr)
while self.running:
for client in self.clientList:
message = client.sock.recv(self.server.BUFFSIZE)
if message != None and message != "":
try:
parsedMsg = json.loads(message)
if (int(parsedMsg['command']) == MSG_LOGIN):
if (self.login(parsedMsg['parameters'])):
client.sock.send('{"result":True,"message":"Login successful."}')
elif (self.loggedIn == True):
client.sock.send(self.parseMessage(parsedMsg))
except Exception as e:
client.sock.send(str(e))
def run(self):
logShim = DBShim(self.params["LOG_DB"], self.keyMgr)
userShim = DBShim(self.params["USER_DB"], self.keyMgr)
keyShim = DBShim(self.params["KEY_DB"], self.keyMgr)
while(True):
print("Audit task trying to verify the rule: " + str(self.rule.config))
# Parse the audit rule
if (self.rule.role != None and self.rule.action != None):
if (self.rule.object == None):
valueMap = {"action" : self.rule.action}
else:
valueMap = {"action" : self.rule.action, "object" : self.rule.object}
eventResults = logShim.executeMultiQuery("Event", valueMap, [])
print(eventResults)
for event in eventResults:
userId = event["userId"]
valueMap = {"roleId" : self.rule.role}
userResults = userShim.executeMultiQuery("UserRole", valueMap, [])
for user in userResults:
if (int(user["userId"]) == int(userId)):
print("Rule violation by user " + str(userId))
time.sleep(15)
class VerifyCrawler(threading.Thread):
''' This is an active thread that is responsible for serving all
messages that come in from the keylogger. It simply strips
them out of the socket and forwards them along to the logger
actor via a message dictionary.
'''
def __init__(self, vid, logServer, keyServer, keyMgr):
''' Constructor that stores the log server information.
'''
threading.Thread.__init__(self)
self.id = vid
self.logServer = logServer
self.keyServer = keyServer
self.running = True
# Build the encryption module
self.keyMgr = keyMgr
self.encryptionModule = EncryptionModule(keyMgr) # pass along the key manager reference
# Generate the used entry bucket
self.usedBin = {}
self.MAX_TRIES = 10 # This can (and should) be configured by experimentation.
# Configure the logger
logFile = 'abls.log'
logging.basicConfig(filename=logFile,level=logging.DEBUG)
def run(self):
''' The main thread loop for this verifier.
'''
# Create the shim
self.logShim = DBShim(self.logServer, self.keyMgr)
self.keyShim = DBShim(self.keyServer, self.keyMgr)
# Run the crawler loop indefinitely...
while self.running:
logging.debug("Verifier " + str(self.id) + " is trying to grab a user session to verify.")
(userId, sessionId) = self.selectRow()
# Check to see if we found some valid data...
if (userId != -1 and sessionId != -1):
# Decrypt the user/session ID to get the original data
userCT = userId.decode("hex")
sessionCT = sessionId.decode("hex")
key = hashlib.sha256(self.keyMgr.getMasterKey() + "log").digest()
cipher = AES.new(key, AES.MODE_ECB)
userPT = cipher.decrypt(userCT)
sessionPT = cipher.decrypt(sessionCT)
# Query the keys from the database
logging.debug("Verifying: " + str(userPT) + " - " + str(sessionPT))
valueMap = {"userId" : userPT, "sessionId" : sessionPT}
epochKey = self.keyShim.executeMultiQuery("initialEpochKey", valueMap, ["userId", "sessionId"])
key1 = epochKey[0]["key"]
entityKey = self.keyShim.executeMultiQuery("initialEntityKey", valueMap, ["userId", "sessionId"])
key2 = entityKey[0]["key"]
# Decrypt the keys using the 'verifier' policy
logging.debug("Trying to decrypt")
sk = self.encryptionModule.generateUserKey(['VERIFIER'])
k1 = self.encryptionModule.decrypt(sk, key1)[1] # [1] to pull out plaintext, [0] is T/F flag
k2 = self.encryptionModule.decrypt(sk, key2)[1] # [1] to pull out plaintext, [0] is T/F flag
# Query the last digest from the database
logging.debug("Decryption successful - continue with the verification process")
entityDigest = self.logShim.executeMultiQuery("entity", valueMap, ["userId", "sessionId"])
digest = entityDigest[len(entityDigest) - 1]["digest"]
# Query for the log now.
valueMap = {"userId" : userId, "sessionId" : sessionId}
logResult = self.logShim.executeMultiQuery("log", valueMap, [])
log = {}
userId = int(userPT)
sessionId = int(sessionPT)
log[(userId, sessionId)] = []
for i in range(0, len(logResult)):
log[(userId, sessionId)].append([userId, sessionId, logResult[i]["epochId"], logResult[i]["message"], logResult[i]["xhash"], logResult[i]["yhash"]])
# Verify the data extracted from the database...
self.strongestVerify(userId, sessionId, log, k1, k2, digest, Logger.Logger.EPOCH_WINDOW_SIZE)
# Don't hog the system resources
time.sleep(15)
def selectRow(self):
''' Randomly select a row from the database to check with strong verification.
'''
userId = sessionId = 0
foundNewRow = False
tries = 0
while not foundNewRow:
result = self.logShim.randomQuery("log")
if (len(result) > 0):
userId = result[0]["userId"]
sessionId = result[0]["sessionId"]
if not ((userId, sessionId) in self.usedBin):
self.usedBin[(userId, sessionId)] = 0
foundNewRow = True
# Upgrade all the instances for
for key in self.usedBin.keys():
self.usedBin[key] = self.usedBin[key] + 1
# See if we ran past the try cap
tries = tries + 1
if (tries >= self.MAX_TRIES):
tk1, tk2, maxNum = 0, 0, 0
for (k1, k2) in self.usedBin.keys():
if (self.usedBin[(k1, k2)] > maxNum):
maxNum = self.usedBin[(k1, k2)]
tk1 = -1
tk2 = -1
del self.usedBin[(tk1, tk2)]
userId = tk1
sessionId = tk2
foundNewRow = True # we're going to retry a previous row
else:
userId = sessionId = -1
foundNewRow = True
return (userId, sessionId)
def strongestVerify(self, userId, sessionId, log, epochKey, entityKey, lastDigest, EPOCH_WINDOW_SIZE = Logger.Logger.EPOCH_WINDOW_SIZE):
''' Walks the log chain and epoch chain for verification, and computes the
entity digests at every epoch cycle for comparison to check with
the end result. Not publicly verifiable, and requires the initial epoch and entity keys.
'''
ctChain = []
sha3 = Keccak.Keccak()
# It is assumed that we would get this initial key from the trusted server...
# This verification scheme is not possible without the epoch key...
lastEpochDigest = hmac.new(epochKey, "0", hashlib.sha512).hexdigest()
# Check to see if we even have anything to verify
if not ((userId, sessionId) in log):
return None
else:
# Handle the base of the chain
first = log[(userId, sessionId)][0]
firstPayload = str(userId) + str(sessionId) + str(0) + str(first[3]) + str(0)
# Check the hash chain first
xi = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex")))
computedV = sha3.Keccak((len(xi), xi))
assert(xi == first[4])
# Check the epoch chain next
yi = hmac.new(epochKey, lastEpochDigest.encode("hex") + first[4].encode("hex"), hashlib.sha512).hexdigest()
assert(yi == first[5])
# Compute the first part of the entity chain now
lastEntityDigest = hmac.new(entityKey, xi, hashlib.sha512).hexdigest()
entityKey = hmac.new(entityKey, "some constant value", hashlib.sha512).hexdigest()
# Append the first message.
ctChain.append(first[3])
# Walk the chain and make sure we can verify it...
for i in range(1, len(log[(userId, sessionId)])):
first = log[(userId, sessionId)][i]
# Store the message
firstMessage = first[3] # the message
ctChain.append(firstMessage)
# The other data...
currentHash = first[4] # the hash
previousHash = log[(userId, sessionId)][i - 1][4]
# Verify that the first entry is correct
firstPayload = str(userId) + str(0) + str(i) + str(firstMessage) + str(previousHash)
firstComputedHash = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex")))
assert(currentHash == firstComputedHash)
# Check the epoch chain to see if we need to cycle
if ((i % EPOCH_WINDOW_SIZE) == 0):
# Update the epoch key
currKey = epochKey
newKey = sha3.Keccak((len(bytes(currKey)), currKey.encode("hex")))
epochKey = newKey
# Pull the last hash block
length = len(log[(userId, sessionId)])
lastHash = log[(userId, sessionId)][i - 1][4]
# Form the epoch block hash payload
payload = str(lastEpochDigest) + str(lastHash)
lastEpochDigest = hmac.new(newKey, payload, hashlib.sha512).hexdigest()
# Compute the epoch chain value
yi = hmac.new(epochKey, lastEpochDigest.encode("hex") + first[4].encode("hex"), hashlib.sha512).hexdigest()
assert(yi == first[5])
# Compute the first part of the entity chain now
lastEntityDigest = hmac.new(entityKey, first[4], hashlib.sha512).hexdigest()
entityKey = hmac.new(entityKey, "some constant value", hashlib.sha512).hexdigest()
assert(lastEntityDigest == lastDigest)
logging.debug("Verification result:" + str(lastEntityDigest == lastDigest))
return ctChain
def weakVerify(self, userId, sessionId, log, epochKey, entityKey, EPOCH_WINDOW_SIZE):
''' Only walks the log chain for verification.
'''
ctChain = []
# Make sure we have something to verify first...
if not ((userId, sessionId) in log):
return None
else:
# Handle the base of the chain
first = log[(userId, sessionId)][0]
firstPayload = str(userId) + str(sessionId) + str(0) + str(first[3]) + str(0)
digest = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex")))
assert(digest == first[4])
# Append the first message.
ctChain.append(first[3])
# Walk the chain and make sure we can verify it...
for i in range(1, len(log[(userId, sessionId)])):
first = log[(userId, sessionId)][i]
# Store the message
firstMessage = first[3] # the message
ctChain.append(firstMessage)
# The other data...
currentHash = first[4] # the hash
previousHash = log[(userId, sessionId)][i - 1][4]
# Verify that the first entry is correct
firstPayload = str(userId) + str(0) + str(i) + str(firstMessage) + str(previousHash)
firstComputedHash = sha3.Keccak((len(bytes(firstPayload)), firstPayload.encode("hex")))
assert(currentHash == firstComputedHash)
return ctChain
def bootstrap():
''' Bootstrap the database from the database with some dummy data.
'''
keyMgr = KeyManager(".")
# Wipe the data if we're in debug mode
print("Debug: Clearing the log database")
# Check to see if we need to clear the table
# This is specific to SQLite - coupling needs to be removed
shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/log.db", keyMgr)
tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='log'")
if (len(tableResults) != 0):
shim.executeRawQuery("DELETE FROM log")
tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='entity'")
if (len(tableResults) != 0):
shim.executeRawQuery("DELETE FROM entity")
tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='epoch'")
if (len(tableResults) != 0):
shim.executeRawQuery("DELETE FROM epoch")
# Check to see if we need to clear the table
# This is specific to SQLite - it needs to be less coupled to SQLite
shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/users.db", keyMgr)
tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='users'")
if (len(tableResults) != 0):
# Delete the contents in the table
shim.executeRawQuery("DELETE FROM users")
print("Initializing dummy data into the users table")
# Create some dummy data for the database
date = datetime.now()
shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["alice", "*****@*****.**", "one", date, date], [False, False, False, False, False])
shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["bob", "*****@*****.**", "two", date, date], [False, False, False, False, False])
shim.insertIntoTable("users", "(name, email, attributes, inserted_at, modified_at)", ["chris", "*****@*****.**", "three", date, date], [False, False, False, False, False])
# Check to see if we need to clear the table
# This is specific to SQLite - it needs to be less coupled to SQLite
shim = DBShim("/Users/caw/Projects/SecureLoggingSystem/src/v1/DatabaseModule/audit_users.db", keyMgr)
tableResults = shim.executeRawQuery("SELECT name FROM sqlite_master WHERE type='table' AND name='audit_users'")
if (len(tableResults) != 0):
# Delete the contents in the table
shim.executeRawQuery("DELETE FROM audit_users")
print("Initializing dummy data into the users table")
# Create some dummy data for the database
date = datetime.now()
alicePassword = "******"
bobPassword = "******"
chrisPassword = "******"
# Generate the passwords/salts
salt = str(uuid.uuid4())
hashed_password = hashlib.sha512(hashlib.sha512(alicePassword).hexdigest() + salt).hexdigest()
shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["alice", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False])
salt = str(uuid.uuid4())
hashed_password = hashlib.sha512(hashlib.sha512(bobPassword).hexdigest() + salt).hexdigest()
shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["bob", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False])
salt = str(uuid.uuid4())
hashed_password = hashlib.sha512(hashlib.sha512(chrisPassword).hexdigest() + salt).hexdigest()
shim.insertIntoTable("audit_users", "(userName, email, password, salt, inserted_at, modified_at)", ["chris", "*****@*****.**", hashed_password, salt, date, date], [False, False, False, False, False, False])
class AuditClientHandler(threading.Thread):
''' This is an active thread that is responsible for serving all
messages that come in from audit proxy.
'''
def __init__(self, serv, params, keyMgr):
''' Initialize the client handler with the parent server (AuditProxy)
'''
threading.Thread.__init__(self)
self.server = serv
self.clientList = []
self.running = True
# Setup the Python logger
logFile = 'abls.log'
logging.basicConfig(filename=logFile,level=logging.DEBUG)
# Persist the key manager and params
self.keyMgr = keyMgr
self.params = params
# Maintain login state of the user is logged in for this session
self.loggedIn = False
def run(self):
''' The main loop for this client handler thread. Strip out a message,
parse it according to the protocol, and then invoke the necessary commands.
'''
global MSG_LOGIN
self.shim = DBShim(self.params["AUDIT_USER_DB"], self.keyMgr)
self.log = DBShim(self.params["LOG_DB"], self.keyMgr)
while self.running:
for client in self.clientList:
message = client.sock.recv(self.server.BUFFSIZE)
if message != None and message != "":
try:
parsedMsg = json.loads(message)
if (int(parsedMsg['command']) == MSG_LOGIN):
if (self.login(parsedMsg['parameters'])):
client.sock.send('{"result":True,"message":"Login successful."}')
elif (self.loggedIn == True):
client.sock.send(self.parseMessage(parsedMsg))
except Exception as e:
client.sock.send(str(e))
def login(self, params):
''' Handle the user login process.
'''
# Strip out the username and password
userNameIndex = params.find(',')
if (userNameIndex != -1):
userName = params[0:userNameIndex]
password = params[userNameIndex + 1:] # format: username,password
# Fetch the salt from the database (there should only be one user by this name)
record = self.shim.executeQuery("audit_users", "userName", userName, False)
if (len(record) == 1):
salt = record[0]["salt"]
hashed_password = hashlib.sha512(password + salt).hexdigest()
if (hashed_password == record[0]["password"]):
self.loggedIn = True
return True
else:
raise Exception("Invalid password for user: "******"Could not find the specified user: "******"client message: " + str(message)) # debug
# Verify that the incoming message conforms to the protocol
if (len(message) != 2):
raise Exception("Invalid JSON string retrieved from client.")
if not (('command' in message) and ('parameters' in message)):
raise Exception("Invalid JSON string retrieved from client.")
# Let it rip
try:
return self.execute(int(message['command']), message['parameters'])
except:
print("Error occured when executing the command")
def execute(self, command, parameters):
''' Execute the command (if possible), or throw an exception if it's invalid.
'''
# Bring the protocol event IDs into scope
global MSG_SELECT_BY_USER
global MSG_SELECT_BY_USER_SESSION
result = ""
# Handle the incoming message
if (command == MSG_SELECT_BY_USER):
#print("MSG_SELECT_BY_USER")
#print(parameters[0])
valueMap = {"userId" : parameters[0]}
rowMasks = ["userId"]
try:
results = self.log.executeMultiQuery("log", valueMap, rowMasks)
# Format the results to return only the ciphertext
logList = []
for i in range(0, len(results)):
logList.append(results[i][4]) # these are encoded in "hex" - decode with .decode("hex")
return json.dumps({"result" : True, "message" : json.dumps(logList)})
except Exception as e:
print(e)
elif (command == MSG_SELECT_BY_USER_SESSION):
#print("MSG_SELECT_BY_USER_SESSION")
#print(parameters[0])
valueMap = {"userId" : entry.userId, "sessionId" : entry.sessionId}
rowMasks = ["userId", "sessionId"]
try:
results = self.log.executeMultiQuery("log", valueMap, rowMasks)
# Format the results to return only the ciphertext
logList = []
for i in range(0, len(results)):
logList.append(results[i][4]) # these are encoded in "hex" - decode with .decode("hex")
return json.dumps({"result" : True, "message" : json.dumps(logList)})
except:
print(e)
else:
raise Exception("Unsupported audit command ID: " + str(command))
# Default return (if other messages don't work)
return json.dumps({"result" : False, "message" : "Error parsing message: " + str(command)})
