def __init__(self, rootDirectory, indexFile, connSocket, connSocketAddress, authHandler):
threading.Thread.__init__(self)
self.rootDirectory = rootDirectory
self.indexFile = indexFile
self.authHandler = authHandler
self.connSocket = connSocket
self.connSocket.settimeout(1)
self.connSocketAddress = connSocketAddress
self.keep_alive = True
self.logger = Logger(self.__class__.__name__+"_{}".format(self.connSocketAddress))
class Scheduler:
def __init__(self, max_threads=50):
self.max_threads = max_threads
self.tasks = []
self.logger = Logger(self.__class__.__name__)
def add(self, runnable):
"""
Add runnable class object to thread queue
"""
self.logger.info("New thread {} added".format(
runnable.connSocketAddress))
self.tasks.append(runnable)
self._update()
def _update(self):
"""
Update thread pool
1. Check for dead threads and remove them
2. Start running new threads that are waiting
"""
# get dead threads
to_delete = []
for runnable in self.tasks:
if not runnable.keep_alive:
to_delete.append(runnable)
self.logger.info("Dead thread {} removed".format(
runnable.connSocketAddress))
# remove dead threads
for task in to_delete:
self.tasks.remove(task)
# start waiting threads, at most max_threads
for runnable in self.tasks[:self.max_threads]:
if not runnable.is_alive():
runnable.start()
self.logger.info("Thread {} started".format(
runnable.connSocketAddress))
def shutdown(self):
"""
Shutdown all running threads
"""
for runnable in self.tasks:
if runnable.is_alive():
runnable.stop()
runnable.join()
self.logger.info("Thread {} stopped".format(
runnable.connSocketAddress))
self.logger.close()
def __init__(self,
rootDirectory,
port,
indexFile="index.html",
enableSSL=False):
# check arguments
if not os.path.exists(rootDirectory):
raise ValueError(
"rootDirectory: {} not found".format(rootDirectory))
self.rootDirectory = os.path.abspath(rootDirectory)
if port > 65535 or port < 0:
raise ValueError("port: {} should be in [0,65535]".format(port))
self.port = port
if not os.path.isfile(os.path.join(self.rootDirectory, indexFile)):
raise ValueError("indexFile: {} is not found under {}".format(
indexFile, self.rootDirectory))
self.indexFile = indexFile
# initialize variables
self.scheduler = Scheduler()
self.logger = Logger(self.__class__.__name__)
# log information
self.logger.info("Server port: {}".format(self.port))
self.logger.info("Server document root: {}".format(self.rootDirectory))
# load authentication handler
self.authHandler = AuthHandler(self.rootDirectory)
# try to load SSL certificate
self.SSL_cert_file = os.path.join("certificates", "signed.crt")
self.SSL_key_file = os.path.join("certificates", "signed.private.key")
if os.path.isfile(self.SSL_cert_file) and os.path.isfile(
self.SSL_key_file) and enableSSL:
self.SSL_context = ssl.create_default_context(
ssl.Purpose.CLIENT_AUTH)
self.SSL_context.load_cert_chain(self.SSL_cert_file,
self.SSL_key_file)
self.SSL_enabled = True
self.logger.info("Server SSL enabled")
else:
self.SSL_context = None
self.SSL_cert_file = ""
self.SSL_key_file = ""
self.SSL_enabled = False
if self.SSL_enabled:
self.logger.info("Website address is: https://{}:{}".format(
"localhost", self.port))
else:
self.logger.info("Website address is: http://{}:{}".format(
"localhost", self.port))
def __init__(self, rootDirectory):
self._init_keys()
self.rootDirectory = rootDirectory
self.logger = Logger(self.__class__.__name__)
self._init_rules()
self.mutex = threading.Condition() # mutex for multithreading synchronization
class AuthHandler:
def __init__(self, rootDirectory):
self._init_keys()
self.rootDirectory = rootDirectory
self.logger = Logger(self.__class__.__name__)
self._init_rules()
self.mutex = threading.Condition() # mutex for multithreading synchronization
def _init_keys(self):
"""
Init keys for rules
"""
self.KEY_Allow = "Allow"
self.KEY_Forbidden = "Forbidden"
self.KEY_Exception = "Exception"
self.KEY_Database = "Database"
self.KEY_Username = "******"
self.KEY_Files = "Files"
self.KEY_Handler = "Handler"
def _init_rules(self):
"""
Init pre-defined rules in root directory
"""
rules = {}
# database = {}
if not os.path.exists(os.path.join(self.rootDirectory, "rules.json")):
# if not found, generate default rules
rules = {
# allow paths
self.KEY_Allow: ["*"],
# forbidden paths, overriden by allowed path
self.KEY_Forbidden: ["*"],
# user-specific exception paths
self.KEY_Exception: [],
# database
self.KEY_Database: "",
# define specific handlers for web page
self.KEY_Handler: {}
}
else:
with open(os.path.join(self.rootDirectory, "rules.json")) as inFile:
rules = json.load(inFile)
# validate rules
if self.KEY_Allow not in rules.keys():
self.logger.warn("'{}' not defined in {}, setting to default".format(self.KEY_Allow, os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Allow] = ["*"]
if self.KEY_Forbidden not in rules.keys():
self.logger.warn("'{}' not defined in {}, setting to default".format(self.KEY_Forbidden, os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Forbidden] = ["*"]
if self.KEY_Exception not in rules.keys():
self.logger.warn("'{}' not defined in {}, setting to default".format(self.KEY_Exception, os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Exception] = []
if self.KEY_Database not in rules.keys():
self.logger.warn("'{}' not defined in {}, setting to default".format(self.KEY_Database, os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Database] = ""
if self.KEY_Handler not in rules.keys():
self.logger.warn("'{}' not defined in {}, setting to default".format(self.KEY_Handler, os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Handler] = {}
# remove wrong format exceptions
rulesExceptionToRemove = []
for item in rules[self.KEY_Exception]:
if (not self.KEY_Username in item.keys()) or (not self.KEY_Files in item.keys()):
self.logger.warn("Item {} has wrong format, removed in {}".format(item, os.path.join(self.rootDirectory, "rules.json")))
rulesExceptionToRemove.append(item)
for item in rulesExceptionToRemove:
rules[self.KEY_Exception].remove(item)
# check database
if not os.path.exists(os.path.join(self.rootDirectory, rules[self.KEY_Database])):
self.logger.warn("Database {} not found, removed in {}".format(os.path.join(self.rootDirectory, rules[self.KEY_Database]), os.path.join(self.rootDirectory, "rules.json")))
rules[self.KEY_Database] = ""
self.databasePath = None
else:
self.databasePath = os.path.join(self.rootDirectory, rules[self.KEY_Database])
# remove not found handlers
rulesHandlerToRemove = []
for key, val in rules[self.KEY_Handler].items():
if (not os.path.isfile(os.path.join(self.rootDirectory, val))) or (val.split(".")[-1] != "py"):
self.logger.warn("Handler {} not found or not Python script, removed in {}".format(os.path.join(self.rootDirectory, val), os.path.join(self.rootDirectory, "rules.json")))
rulesHandlerToRemove.append(key)
for key in rulesHandlerToRemove:
del rules[self.KEY_Handler][key]
self.rules = rules
self.logger.info("Rules initialized")
self.authorized_list = {}
self.logger.info("Authorization list initialized")
self._save()
def _save(self):
"""
Save updated rules
"""
with open(os.path.join(self.rootDirectory, "rules.json"), "w") as outFile:
json.dump(self.rules, outFile, indent=4)
self.logger.info("Rules saved")
def auth(self, path, clientIP):
"""
Authenticate path, given a user
"""
# remove port information
clientIP = clientIP.split(":")[0]
if clientIP in self.authorized_list.keys():
user = self.authorized_list[clientIP]
# if user is given and database is not empty, check exception
if user and self.databasePath:
for item in self.rules[self.KEY_Exception]:
if item[self.KEY_Username] == user:
# check if path is exception
for filename in item[self.KEY_Files]:
# user glob for path matching
if os.path.normpath(path) in glob.glob(os.path.join(self.rootDirectory, filename)):
return True # access is accepted
break
# else check in allowed paths
for item in self.rules[self.KEY_Allow]:
if os.path.normpath(path) in glob.glob(os.path.join(self.rootDirectory, item)):
return True
# else check forbidden paths
for item in self.rules[self.KEY_Forbidden]:
if os.path.normpath(path) in glob.glob(os.path.join(self.rootDirectory, item)):
return False
# by default, return True
self.logger.warn("Path {} is authenticated, but not mentioned in rules.json".format(path))
return True
def handle(self, path, params):
"""
Handle parameters using specified handlers, only for html pages
"""
if not params: return []
pathHead, pathTail = ntpath.split(path)
filename = pathTail or ntpath.basename(pathHead)
for key, val in self.rules[self.KEY_Handler].items():
if key == filename:
command = "python \"{}\" ".format(glob.glob(os.path.join(self.rootDirectory, val))[0])
for pKey, pVal in params.items():
command += "--{} ".format(pKey)
for pValItem in pVal:
command += "\"{}\" ".format(pValItem)
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = proc.communicate()[0]
if len(result) <= 0: return []
result = result.decode("utf-8").split("\r\n\r\n") # read output from handler file and convert to string
result = [x for x in result if x]
return result
self.logger.warn("Failed to handle {}, unknown handler".format(path))
return []
def updateUserSession(self, clientIP, user):
"""
Update authorized user session
"""
# remove port information
clientIP = clientIP.split(":")[0]
self.authorized_list[clientIP] = user
def shutdown(self):
"""
Shutdown authentication handler and save updated information and log content
"""
self._save()
self.logger.close()
def __init__(self, max_threads=50):
self.max_threads = max_threads
self.tasks = []
self.logger = Logger(self.__class__.__name__)
class Server:
def __init__(self,
rootDirectory,
port,
indexFile="index.html",
enableSSL=False):
# check arguments
if not os.path.exists(rootDirectory):
raise ValueError(
"rootDirectory: {} not found".format(rootDirectory))
self.rootDirectory = os.path.abspath(rootDirectory)
if port > 65535 or port < 0:
raise ValueError("port: {} should be in [0,65535]".format(port))
self.port = port
if not os.path.isfile(os.path.join(self.rootDirectory, indexFile)):
raise ValueError("indexFile: {} is not found under {}".format(
indexFile, self.rootDirectory))
self.indexFile = indexFile
# initialize variables
self.scheduler = Scheduler()
self.logger = Logger(self.__class__.__name__)
# log information
self.logger.info("Server port: {}".format(self.port))
self.logger.info("Server document root: {}".format(self.rootDirectory))
# load authentication handler
self.authHandler = AuthHandler(self.rootDirectory)
# try to load SSL certificate
self.SSL_cert_file = os.path.join("certificates", "signed.crt")
self.SSL_key_file = os.path.join("certificates", "signed.private.key")
if os.path.isfile(self.SSL_cert_file) and os.path.isfile(
self.SSL_key_file) and enableSSL:
self.SSL_context = ssl.create_default_context(
ssl.Purpose.CLIENT_AUTH)
self.SSL_context.load_cert_chain(self.SSL_cert_file,
self.SSL_key_file)
self.SSL_enabled = True
self.logger.info("Server SSL enabled")
else:
self.SSL_context = None
self.SSL_cert_file = ""
self.SSL_key_file = ""
self.SSL_enabled = False
if self.SSL_enabled:
self.logger.info("Website address is: https://{}:{}".format(
"localhost", self.port))
else:
self.logger.info("Website address is: http://{}:{}".format(
"localhost", self.port))
def start(self):
# create server socket
try:
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(("", self.port))
serversocket.listen(5)
serversocket.settimeout(1)
except socket.error as e:
self.logger.error("{}".format(e))
return
self.logger.info("Server started")
# start listening
try:
while True:
try:
clientsocket, clientaddress = serversocket.accept()
clientaddress = "{}:{}".format(clientaddress[0],
clientaddress[1])
if self.SSL_enabled:
clientsocket = self.SSL_context.wrap_socket(
clientsocket, server_side=True)
self.logger.info(
"Client connected: {}".format(clientaddress))
processor = RequestProcessor(self.rootDirectory,
self.indexFile, clientsocket,
clientaddress,
self.authHandler)
self.scheduler.add(processor)
except socket.timeout:
pass
except ssl.SSLError as e:
# ignore HTTP request error in HTTPS mode
self.logger.error(e)
except ConnectionAbortedError as e:
self.logger.error(e)
except KeyboardInterrupt:
# on keyboard interrupt, close server and all running sub-threads
self.logger.info("Server stopped")
self.scheduler.shutdown()
self.authHandler.shutdown()
self.logger.close()
serversocket.close()
class RequestProcessor(threading.Thread):
def __init__(self, rootDirectory, indexFile, connSocket, connSocketAddress, authHandler):
threading.Thread.__init__(self)
self.rootDirectory = rootDirectory
self.indexFile = indexFile
self.authHandler = authHandler
self.connSocket = connSocket
self.connSocket.settimeout(1)
self.connSocketAddress = connSocketAddress
self.keep_alive = True
self.logger = Logger(self.__class__.__name__+"_{}".format(self.connSocketAddress))
def run(self):
"""
Start processing requests (called by thread scheduler)
"""
while self.keep_alive:
# recieve data from client socket
try:
received = self.connSocket.recv(2048)
except socket.timeout: continue
except socket.error as e:
self.logger.error(e)
self.stop()
break
# decode byte array to string (HTTP request)
try:
decodedMessage = received.decode("utf-8")
except UnicodeDecodeError as e:
self.logger.error(e)
self.stop()
break
# handle request
self._handle(decodedMessage)
def stop(self):
"""
Stop processing requests (called by thread scheduler)
"""
self.keep_alive = False
self.connSocket.detach()
self.logger.close()
def _handle(self, request):
"""
Handle received request message (POST, GET, HEAD)
"""
# if empty request, just skip
if not request.strip():
return
# recognize message type
type = request.split()[0].lower()
# handle POST
if type == "post" :
self._handlePOST(request)
# handle GET
elif type == "get" :
self._handleGET(request)
# handle HEAD
elif type == "head" :
self._handleHEAD(request)
# handle not implemented
else:
self.logger.warn("{} request type is not implemented")
self._handleERROR(501, "Not Implemented")
def _send(self, message, binary=False):
"""
Send response to client after handling\\
Return `True` if success\\
Return `False` if error occured
"""
# set timeout to blocking, for all data to be sent
try:
self.connSocket.settimeout(None)
if binary:
self.connSocket.send(message)
else:
self.connSocket.send(message.encode("utf-8"))
# set back timeout
self.connSocket.settimeout(1)
return True
except socket.error as e:
self.logger.error(e)
return False
def _sendHEADER(self, responseCode, responseMessage, contentType, length):
"""
send http response header
"""
#create each line of the header
header = [
"HTTP/1.1 {} {}\r\n".format(responseCode, responseMessage),
"Date: {}\r\n".format(formatdate(timeval=None, localtime=False, usegmt=True)),
"Server: Pr0j3ct\r\n",
"Content-Length: {}\r\n".format(length),
"Content-Type: {}\r\n".format(contentType),
]
#convert header to a single string
header = "".join(header)
header += "\r\n"
#send header
self._send(header)
def _handleGET(self, message):
"""
handle GET http request
"""
received = message.split("\n")[0]
#convert URL to original string
targetInfoParsed = urllib.parse.urlparse(received.split()[1])
targetInfo = urllib.parse.unquote(targetInfoParsed.path)
targetParams = urllib.parse.parse_qs(targetInfoParsed.query)
self.logger.info("GET {}".format(targetInfo))
#if requested root send back index file
if targetInfo == "/":
self.authHandler.mutex.acquire()
data = self.authHandler.handle(os.path.join(self.rootDirectory, self.indexFile), targetParams)
self.authHandler.mutex.release()
if len(data) <= 0:
with open(os.path.join(self.rootDirectory, self.indexFile), "r") as inputFile:
data = inputFile.read()
self._sendHEADER(200, "OK", "text/html; charset=utf-8", len(data))
self._send(data)
elif len(data) == 1:
self._send(data[0] + "\r\n\r\n")
else:
self._send(data[0] + "\r\n\r\n")
self._send(data[1])
#else try to recognize target file
else:
# convert to relative target path
targetInfo = "." + targetInfo
#get abosolute filepath
filePath = os.path.join(self.rootDirectory, targetInfo)
# if path not exist, send 404 error
if not os.path.exists(filePath):
self.logger.warn("GET {} is not a path".format(filePath))
self._handleERROR(404, "File Not Found")
# if request target is not a file, send 404 error.
elif not os.path.isfile(filePath):
self.logger.warn("GET {} is not a file".format(filePath))
self._handleERROR(404, "File Not Found")
#if requested file is out of the root directory, send permission denied.
elif os.path.commonpath([self.rootDirectory]) != os.path.commonpath([self.rootDirectory, filePath]):
self.logger.warn("GET {} not in root directory".format(filePath))
self._handleERROR(403, "Permission Denied")
# else send back requested file
else:
self.authHandler.mutex.acquire()
authorized = self.authHandler.auth(filePath, self.connSocketAddress)
self.authHandler.mutex.release()
# if not authorized
if not authorized:
self.logger.warn("GET {} not authorized".format(filePath))
self._handleERROR(403, "Permission Denied")
return
self.authHandler.mutex.acquire()
data = self.authHandler.handle(filePath, targetParams)
self.authHandler.mutex.release()
if len(data) <= 0:
# get file size in bytes
fileSize = os.path.getsize(filePath)
# get data type
datatype, _ = mimetypes.guess_type(filePath)
if not datatype:
# if not able to guess, set to "application/octet-stream" (default binary file type)
self.logger.warn("GET {} unknown mime type, set to application/octet-stream".format(filePath))
datatype = "application/octet-stream"
# send header and data
self._sendHEADER(200, "OK", datatype, fileSize)
with open(filePath, "rb") as inputFile:
while True:
# read every 2048 bytes
data = inputFile.read(2048)
# if no more data, stop
if not data: break
# if send data failed, break
if not self._send(data, binary=True):
self.logger.warn("GET {} failed to send".format(filePath))
break
elif len(data) == 1:
self._send(data[0] + "\r\n\r\n")
else:
self._send(data[0] + "\r\n\r\n")
self._send(data[1])
def _handleHEAD(self, message):
"""
handle HEAD http request
"""
received = message.split("\n")[0]
#convert URL to original string
targetInfoParsed = urllib.parse.urlparse(received.split()[1])
targetInfo = urllib.parse.unquote(targetInfoParsed.path)
targetParams = urllib.parse.parse_qs(targetInfoParsed.query)
self.logger.info("HEAD {}".format(targetInfo))
#if requested root send back index file header
if targetInfo == "/" :
self.authHandler.mutex.acquire()
data = self.authHandler.handle(os.path.join(self.rootDirectory, self.indexFile), targetParams)
self.authHandler.mutex.release()
if len(data) <= 0:
with open(os.path.join(self.rootDirectory, self.indexFile), "r") as inputFile:
data = inputFile.read()
self._sendHEADER(200, "OK", "text/html; charset=utf-8", len(data))
elif len(data) == 1:
self._send(data[0] + "\r\n\r\n")
else:
self._send(data[0] + "\r\n\r\n") # first is header information
else:
# convert to relative target path
targetInfo = "." + targetInfo
#get abosolute filepath
filePath = os.path.join(self.rootDirectory, targetInfo)
# if path not exist, send 404 error
if not os.path.exists(filePath):
self.logger.warn("HEAD {} is not a path".format(filePath))
self._handleERROR(404, "File Not Found", nobody=True)
# if request target is not a file, send 404 error.
elif not os.path.isfile(filePath):
self.logger.warn("HEAD {} is not a file".format(filePath))
self._handleERROR(404, "File Not Found", nobody=True)
#if requested file is out of the root directory, send permission denied.
elif os.path.commonpath([self.rootDirectory]) != os.path.commonpath([self.rootDirectory, filePath]):
self.logger.warn("HEAD {} not in root directory".format(filePath))
self._handleERROR(403, "Permission Denied", nobody=True)
# else send back requested file
else:
self.authHandler.mutex.acquire()
authorized = self.authHandler.auth(filePath, self.connSocketAddress)
self.authHandler.mutex.release()
# if not authorized
if not authorized:
self.logger.warn("GET {} not authorized".format(filePath))
self._handleERROR(403, "Permission Denied", nobody=True)
return
self.authHandler.mutex.acquire()
data = self.authHandler.handle(filePath, targetParams)
self.authHandler.mutex.release()
if len(data) <= 0:
# get file size in bytes
fileSize = os.path.getsize(filePath)
# get data type
datatype, _ = mimetypes.guess_type(filePath)
if not datatype:
# if not able to guess, set to "application/octet-stream" (default binary file type)
self.logger.warn("HEAD {} unknown mime type, set to application/octet-stream".format(filePath))
datatype = "application/octet-stream"
# send header
self._sendHEADER(200, "OK", datatype, fileSize)
elif len(data) == 1:
self._send(data[0] + "\r\n\r\n")
else:
self._send(data[0] + "\r\n\r\n")
def _handlePOST(self, message):
"""
handle POST http request
"""
# get target info and parameters
targetInfoParsed = urllib.parse.urlparse(message.split("\n")[0].split()[1])
targetInfo = urllib.parse.unquote(targetInfoParsed.path)
targetParams = urllib.parse.parse_qs(message.split("\r\n\r\n")[-1])
#convert URL to original string
targetInfo = urllib.parse.unquote(targetInfo)
self.logger.info("POST {}".format(targetInfo))
# handle parameters
self.authHandler.mutex.acquire()
data = self.authHandler.handle(os.path.join(self.rootDirectory, targetInfo), targetParams)
self.authHandler.mutex.release()
if len(data) <= 0:
self.logger.warn("POST {} is not handled".format(targetInfo))
self._handleERROR(501, "Not Supported")
elif len(data) == 1:
#if is login page, do authentication as well
if (targetInfo.lower() == "/login.html") and "username" in targetParams.keys():
# specific case, len(data) == 1 means login success
self.logger.info("login successful")
self.authHandler.mutex.acquire()
self.authHandler.updateUserSession(self.connSocketAddress, targetParams["username"][0])
self.authHandler.mutex.release()
self._send(data[0] + "\r\n\r\n")
else:
if targetInfo.lower() == "/login.html":
self.logger.warn("login not successful")
self.authHandler.mutex.acquire()
self.authHandler.updateUserSession(self.connSocketAddress, None)
self.authHandler.mutex.release()
self._send(data[0] + "\r\n\r\n")
self._send(data[1])
def _handleERROR(self, errorCode, errorMessage, nobody=False):
"""
handle http request ERROR
"""
#create each line for the response html
body = [
"<!DOCTYPE html>\r\n",
"<html>\r\n",
"<head>\r\n",
"<title>{}</title>\r\n",
"</head>\r\n",
"<body>\r\n",
"<h1>HTTP Error {}: {}</h1>\r\n"
"</body>\r\n",
"</html>\r\n",
]
#convert body to a single string
body = "".join(body).format(errorMessage, errorCode, errorMessage)
#send header
self._sendHEADER(errorCode, errorMessage, "text/html; charset=utf-8", len(body))
if not nobody:
#send body
self._send(body)