def ListenForClients(port, messageQueue, taskQueue, IOLock, numberOfConnections, numberOfConnectionsLock,resultsDirectory):
'''Handles connecting clients
This method opens a port to listen for clients. Whenever a new client connects, a new 'HandleConnection' process is started.
Args
- port -- the port on which the server should listen for clients
- messageQueue -- the Queue from which the logger reads
- taskQueue -- the Queue where all the tasks are located
- IOLock -- Lock which needs to be aquired before reading or writing tasks to disc
- numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
- numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
'''
#create a socket
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('',port))
s.listen(1)
messageQueue.put(UtilityFunctions.createLogEntry('inf', 'Listening for clients'))
except socket.error, msg:
print 'foo'
messageQueue.put(UtilityFunctions.createLogEntry('inf','Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1] + format_exc()))
raise
def handleClientTask(messageQueue, serverMessage,resultsDirectory):
'''Processes a task received by the server
Args:
- messageQueue -- queue used for logging
- serverMessge -- message received from server
'''
temporaryDirectory = './client/temp'
if serverMessage['messageType'] == 'task':
#write task to file
clientScript = zlib.decompress(serverMessage['task'])#decompress the
clientScriptHandle = open(os.path.join(temporaryDirectory,'clientTask.py'),'w')
clientScriptHandle.writelines(clientScript)
clientScriptHandle.close()
dataDirectory = None
#if data exists, write data to file and extract it to ./temp/receivedData
if not serverMessage['data'] == None:
dataDirectory = os.path.join(temporaryDirectory,'receivedData')
os.mkdir(dataDirectory)
dataHandle = open(os.path.join(temporaryDirectory,'data.zip'),'w')
dataHandle.writelines(serverMessage['data'])
dataHandle.close()
UtilityFunctions.unzipFile(os.path.join(temporaryDirectory,'data.zip'), dataDirectory)
#execute received script
subprocess.call(['python',os.path.join(temporaryDirectory,'clientTask.py'),os.path.abspath(dataDirectory)
,os.path.abspath(resultsDirectory),])
#cleanup temp folder
UtilityFunctions.deleteContentsOfDirectory(temporaryDirectory)
else:
raise Exception('received an unknown Message type from the server. The type received was: '+serverMessage['task'])
def completeTask(messageQueue, IOLock, task, resultData, resultsDirectory):
'''unzips resultData in results/taskName, writes task into completedTasks.txt
Args:
- messageQueue -- queue used for logging
- IOLock -- the lock that must be acquired when reading or writing data
- task -- the task that was completed
- resultData -- a zip file containing the results from the client loaded into memory as a zip file
'''
IOLock.acquire()
pathToResult = ''
try:
#write results to file
#---------------------
taskName = task.split(',')[0]
pathToResult = os.path.join(resultsDirectory,taskName)
pathToCompletedTasks = os.path.join(resultsDirectory,'completedTasks.txt')
#check if directory already exists, if it does, that means the task name was not unique
if os.path.isdir(pathToResult):
#add a counter to the task
counter = 1
pathToResult = os.path.join(resultsDirectory,taskName+str(counter))
while(os.path.isdir(pathToResult)): #Note that counter will not overflow, as pythons ints are limited by address space, see http://stackoverflow.com/questions/9860588/maximum-value-for-long-integer so there will be other issues long before counter gets to big
counter = counter+1;
pathToResult = os.path.join(resultsDirectory,taskName+str(counter))
messageQueue.put(UtilityFunctions.createLogEntry('err','The following taskname is not unique: '+taskName+' the taskname was stored as: '
+ taskName+str(counter)))
#unzip results into pathToResult
resultHandle = open('./server/temp/results.zip','w')
resultHandle.writelines(resultData)
resultHandle.close()
os.makedirs(pathToResult)
UtilityFunctions.unzipFile('./server/temp/results.zip',pathToResult)
os.remove('./server/temp/results.zip')
#update the completedTasks.txt file in temp to include the completed task
if os.path.isfile(pathToCompletedTasks) and os.path.getsize(pathToCompletedTasks)>0:
completedTasksLogHandle = open(pathToCompletedTasks,'a')
else:
completedTasksLogHandle = open(pathToCompletedTasks,'w')
completedTasksLogHandle.write(task+'\n')
completedTasksLogHandle.close()
messageQueue.put(UtilityFunctions.createLogEntry('inf','Successfuly stored result for task: ' + task))
except:
messageQueue.put(UtilityFunctions.createLogEntry('deb','error in ServerSideTaskHandler.completeTask:\n'+traceback.format_exc()+'\ntask' + task))
raise
finally:
IOLock.release()
return pathToResult
def getTask(messageQueue, taskQueue, IOLock):
'''Takes a task from taskQueue, interprets it, loads instructions and data into memory
Args:
- messageQueue -- queue used for logging
- taskQueue -- the global task queue from
- IOLock -- the lock that must be acquired when reading or writing data
Returns:
- task -- the task itself, if there are no tasks, this will be none
- clientScript -- a python file loaded into memory and compressed with zlib.compress, if there are no tasks, this will be none
- data -- a zipped file containing data for the client loaded into memory, if there are no tasks, this will be none, if no data folder exists, this will be none
Raises:
- Exceptions.NoTasks: if there are no tasks in the task queue
'''
IOLock.acquire()
try:
while True:
try:
task = taskQueue.get(True,0.1)
except Queue.Empty:
return None,None,None
try:
taskName,pathToScript,pathToData = task.split(',')
except ValueError:
messageQueue.put(UtilityFunctions.createLogEntry('err','the following task is in the wrong format: '+task))
continue
try:
scriptHandle = open(pathToScript,'r')
clientScript = zlib.compress(scriptHandle.read());
scriptHandle.close();
except IOError:
messageQueue.put(UtilityFunctions.createLogEntry('err','could not read the following clientScript: '+pathToScript))
continue
if os.path.isdir(pathToData):
UtilityFunctions.zipDir(pathToData, './server/temp/transferData.zip');
transferDataHandle = open('./server/temp/transferData.zip');
data = transferDataHandle.read();
transferDataHandle.close();
os.remove('./server/temp/transferData.zip')
else:
data = None
break
return task,clientScript,data
except:
messageQueue.put(UtilityFunctions.createLogEntry('deb','error in ServerSideTaskHandler.getTask:\n'+traceback.format_exc() + '\ntask' + task));
raise
finally:
IOLock.release()
def getTask(messageQueue, taskQueue, IOLock):
'''Takes a task from taskQueue, interprets it, loads instructions and data into memory
Args:
- messageQueue -- queue used for logging
- taskQueue -- the global task queue from
- IOLock -- the lock that must be acquired when reading or writing data
Returns:
- task -- the task itself, if there are no tasks, this will be none
- clientScript -- a python file loaded into memory and compressed with zlib.compress, if there are no tasks, this will be none
- data -- a zipped file containing data for the client loaded into memory, if there are no tasks, this will be none, if no data folder exists, this will be none
Raises:
- Exceptions.NoTasks: if there are no tasks in the task queue
'''
IOLock.acquire()
try:
while True:
try:
task = taskQueue.get(True,0.1)
except Queue.Empty:
return None,None,None
try:
taskName,pathToScript,pathToData = task.split(',')
except ValueError:
messageQueue.put(UtilityFunctions.createLogEntry('err','the following task is in the wrong format: '+task))
continue
try:
scriptHandle = open(pathToScript,'r')
clientScript = zlib.compress(scriptHandle.read());
scriptHandle.close();
except IOError:
messageQueue.put(UtilityFunctions.createLogEntry('err','could not read the following clientScript: '+pathToScript))
continue
if os.path.isdir(pathToData):
UtilityFunctions.zipDir(pathToData, './server/temp/transferData.zip');
transferDataHandle = open('./server/temp/transferData.zip');
data = transferDataHandle.read();
transferDataHandle.close();
os.remove('./server/temp/transferData.zip')
else:
data = None
break
return task,clientScript,data
except:
messageQueue.put(UtilityFunctions.createLogEntry('deb','error in ServerSideTaskHandler.getTask:\n'+traceback.format_exc() + '\ntask' + task));
raise
finally:
IOLock.release()
def ListenForClients(port, messageQueue, taskQueue, IOLock,
numberOfConnections, numberOfConnectionsLock,
resultsDirectory):
'''Handles connecting clients
This method opens a port to listen for clients. Whenever a new client connects, a new 'HandleConnection' process is started.
Args
- port -- the port on which the server should listen for clients
- messageQueue -- the Queue from which the logger reads
- taskQueue -- the Queue where all the tasks are located
- IOLock -- Lock which needs to be aquired before reading or writing tasks to disc
- numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
- numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
'''
#create a socket
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', port))
s.listen(1)
messageQueue.put(
UtilityFunctions.createLogEntry('inf', 'Listening for clients'))
except socket.error, msg:
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Bind failed. Error Code : ' + str(msg[0]) +
' Message ' + msg[1] + format_exc()))
raise
def completeTask(messageQueue, IOLock, task, resultData, resultsDirectory):
'''unzips resultData in results/taskName, writes task into completedTasks.txt
Args:
- messageQueue -- queue used for logging
- IOLock -- the lock that must be acquired when reading or writing data
- task -- the task that was completed
- resultData -- a zip file containing the results from the client loaded into memory as a zip file
'''
IOLock.acquire()
pathToResult = ''
try:
#write results to file
#---------------------
taskName = task.split(',')[0]
pathToResult = os.path.join(resultsDirectory,taskName)
pathToCompletedTasks = os.path.join(resultsDirectory,'completedTasks.txt')
#check if directory already exists, if it does, that means the task name was not unique
if os.path.isdir(pathToResult):
#add a counter to the task
counter = 1
pathToResult = os.path.join(resultsDirectory,taskName+str(counter))
while(os.path.isdir(pathToResult)): #Note that counter will not overflow, as pythons ints are limited by address space, see http://stackoverflow.com/questions/9860588/maximum-value-for-long-integer so there will be other issues long before counter gets to big
counter = counter+1;
pathToResult = os.path.join(resultsDirectory,taskName+str(counter))
messageQueue.put(UtilityFunctions.createLogEntry('err','The following taskname is not unique: '+taskName+' the taskname was stored as: '
+ taskName+str(counter)))
#unzip results into pathToResult
resultHandle = open('./server/temp/results.zip','w')
resultHandle.writelines(resultData)
resultHandle.close()
os.makedirs(pathToResult)
UtilityFunctions.unzipFile('./server/temp/results.zip',pathToResult)
os.remove('./server/temp/results.zip')
#update the completedTasks.txt file in temp to include the completed task
if os.path.isfile(pathToCompletedTasks) and os.path.getsize(pathToCompletedTasks)>0:
completedTasksLogHandle = open(pathToCompletedTasks,'a')
else:
completedTasksLogHandle = open(pathToCompletedTasks,'w')
completedTasksLogHandle.write(task+'\n')
completedTasksLogHandle.close()
messageQueue.put(UtilityFunctions.createLogEntry('inf','Successfuly stored result for task: ' + task))
except:
messageQueue.put(UtilityFunctions.createLogEntry('deb','error in ServerSideTaskHandler.completeTask:\n'+traceback.format_exc()+'\ntask' + task))
raise
finally:
IOLock.release()
return pathToResult
def clientMain(serverIP, serverPort):
'''Starts the client
Args:
- serverIP -- ip address of server to connect to
- serverPort -- port on which the server is listening
'''
#initialization of values
#timeName = time.strftime("%d%b%Y%H%M%S", time.localtime()) # a unique id created from the time when the script starts
messageQueue = multiprocessing.Queue() # the queue where all log messages are written to.
logControlValue = multiprocessing.Value('i',int(True)) #A boolean indicating the the logging process should continue to run. There is no type for bool, so we use int
resultsDirectory, logsDirectory = UtilityFunctions.initializeClientFolderStructure() #directories where results and logs should be stored respectively
UtilityFunctions.deleteContentsOfDirectory('./client/temp')
#-------------start logging module
loggingProcess = multiprocessing.Process(target = LoggingModule.log, args=(logControlValue,messageQueue,logsDirectory))
loggingProcess.start()
#communicate with server
handleConnection(messageQueue, serverIP,serverPort,resultsDirectory)
#exit note that we only get here once the server closes the connection
logControlValue.value = int(False)
loggingProcess.join()
sys.exit(0)
def clientMain(port, resultsName, pathToTasks, pathToPreviouslyCompletedTasks = None):
'''Starts the server.
Args:
- port -- port on which the server should listen
- resultsName -- name to which to append the currentTime in order to create a unique string
- pathToTasks -- path to the tasks file
- pathToPreviouslyCompletedTasks -- path to a file containing tasks that have already been completed and should not be executed again.
'''
#-------------initialization of values
timeName = resultsName+datetime.datetime.now().strftime('%d%b%Y%H%M%S.%f') # a unique id created from the time when the script starts
messageQueue = multiprocessing.Queue() # the queue where all log messages are written to.
taskQueue = multiprocessing.Queue() # the queue where all tasks that have been loaded into memory and have not been completed are kept
logControlValue = multiprocessing.Value('i',int(True)) #A boolean indicating the the logging process should continue to run. There is no type for bool, so we use int
numberOfConnectionsValue = multiprocessing.Value('i',0) #An integer counting the number of clients that are currently connected to the server
numberOfConnectionsLock = multiprocessing.Lock() #A Lock that should be acquired when modifying the numberOfConnections value
IOLock = multiprocessing.Lock() #A lock that should be acquired when performing writing of results to memory
signal.signal(signal.SIGINT, signal_handler) #register our signal_handler so that we can detect ctrl-C signals
resultsDirectory, logsDirectory = UtilityFunctions.initializeServerFolderStructure(timeName) #directories where results and logs should be stored respectively
if not ServerSideTaskHandler.initializeTaskQueue(taskQueue, pathToTasks,resultsDirectory,pathToPreviouslyCompletedTasks):
sys.exit(1)# initialize Task Queue returns false if initialization and loading of tasks failed
#-------------done initializing
#-------------start logging module
loggingProcess = multiprocessing.Process(target = LoggingModule.log, args=(logControlValue,messageQueue,logsDirectory))
loggingProcess.start()
#Start listening for clients:
#ListenForClients(port,messageQueue,taskHandler) #DEBUG
listeningProcess = multiprocessing.Process(target = ListenForClients, args=(port, messageQueue, taskQueue, IOLock, numberOfConnectionsValue, numberOfConnectionsLock,resultsDirectory))
listeningProcess.start()
#idle while there are clients connected and tasks still incomplete
now = datetime.datetime.now()
logPeriod = datetime.timedelta(minutes=10)
messageQueue.put(UtilityFunctions.createLogEntry('inf','Periodic log entry, there are currently:\t '
+ str(taskQueue.qsize()) + ' tasks yet to complete'))
while numberOfConnectionsValue.value>0 or taskQueue.qsize() > 0:
if datetime.datetime.now() > now+logPeriod:
messageQueue.put(UtilityFunctions.createLogEntry('inf','Periodic log entry, there are currently:\t '
+ str(taskQueue.qsize()) + ' tasks yet to complete'))
now = datetime.datetime.now()
time.sleep(1)
messageQueue.put(UtilityFunctions.createLogEntry('inf','Completed all tasks, exiting'))
listeningProcess.terminate()
logControlValue.value = int(False)
loggingProcess.join()
listeningProcess.join()
sys.exit(0)
def processResult(messageQueue, IOLock, pathToReceivedResult):
'''Server will call this method just after it received and stored a result from a client.
Replace this method with code that should update the data and the clienttasks that have not been
executed so far if this is necessary. By default this method does nothing.
Args:
- messageQueue: -- queue used for logging
- IOLock: -- A lock that should be acquired before any modifications to tasks and data are made to prevent race conditions. Note that if updating tasks takes too long, this will be a bottleneck
- pathToReceivedResult: -- the path to where the most recent task was stored
'''
#This method should eventually implement boosting by updating a vector that
#sends results to
timeName = datetime.datetime.now().strftime('%d%b%Y%H%M%S.%f/')
pathToReceivedResult = pathToReceivedResult + '/..'
pathToSendResult = '/root/dcap_bnet/results/' + timeName
try:
sendResults(messageQueue, pathToReceivedResult, pathToSendResult)
# pass
except Exception as e:
messageQueue.put(
UtilityFunctions.createLogEntry(
'err', "Could not send results to %s because: " %
(pathToSendResult) + str(e) + format_exc()))
def processResult(messageQueue, IOLock, pathToReceivedResult):
'''Server will call this method just after it received and stored a result from a client.
Replace this method with code that should update the data and the clienttasks that have not been
executed so far if this is necessary. By default this method does nothing.
Args:
- messageQueue: -- queue used for logging
- IOLock: -- A lock that should be acquired before any modifications to tasks and data are made to prevent race conditions. Note that if updating tasks takes too long, this will be a bottleneck
- pathToReceivedResult: -- the path to where the most recent task was stored
'''
#This method should eventually implement boosting by updating a vector that
#sends results to
timeName = datetime.datetime.now().strftime('%d%b%Y%H%M%S.%f/')
print os.getcwd()
print "path to received result", pathToReceivedResult
pathToReceivedResult = pathToReceivedResult + '/..'
pathToSendResult = '/root/dcap_bnet/results/' + timeName
try:
sendResults(messageQueue, pathToReceivedResult , pathToSendResult)
# pass
except Exception as e:
messageQueue.put(UtilityFunctions.createLogEntry('err',"Could not send results to %s because: " % (pathToSendResult) + str(e) + format_exc()))
def sendResults(messageQueue, pathToReceivedResult, pathToSendResult):
'''Server sends results to specified location using FTP
Args:
- pathToReceivedResult: -- the path to where the most recent task was stored
- pathToSendResult: -- path where results should be sent
'''
messageQueue.put(UtilityFunctions.createLogEntry('inf','sending results from ' + pathToReceivedResult + ' to ' + pathToSendResult))
shutil.copytree(pathToReceivedResult , pathToSendResult)
def handleClientTask(messageQueue, serverMessage, resultsDirectory):
'''Processes a task received by the server
Args:
- messageQueue -- queue used for logging
- serverMessge -- message received from server
'''
temporaryDirectory = './client/temp'
if serverMessage['messageType'] == 'task':
#write task to file
clientScript = zlib.decompress(serverMessage['task']) #decompress the
clientScriptHandle = open(
os.path.join(temporaryDirectory, 'clientTask.py'), 'w')
clientScriptHandle.writelines(clientScript)
clientScriptHandle.close()
dataDirectory = None
#if data exists, write data to file and extract it to ./temp/receivedData
if not serverMessage['data'] == None:
dataDirectory = os.path.join(temporaryDirectory, 'receivedData')
os.mkdir(dataDirectory)
dataHandle = open(os.path.join(temporaryDirectory, 'data.zip'),
'w')
dataHandle.writelines(serverMessage['data'])
dataHandle.close()
UtilityFunctions.unzipFile(
os.path.join(temporaryDirectory, 'data.zip'), dataDirectory)
#execute received script
subprocess.call([
'python',
os.path.join(temporaryDirectory, 'clientTask.py'),
os.path.abspath(dataDirectory),
os.path.abspath(resultsDirectory),
])
#cleanup temp folder
UtilityFunctions.deleteContentsOfDirectory(temporaryDirectory)
else:
raise Exception(
'received an unknown Message type from the server. The type received was: '
+ serverMessage['task'])
def returnTask(messageQueue, taskQueue, task):
'''If a task fails and needs to be returned, call this method, it returns the task to the taskQueue
Args:
- messageQueue -- queue used for logging
- taskQueue -- the global task queue from
- task -- the task that should be returned
'''
taskQueue.put(task)
messageQueue.put(UtilityFunctions.createLogEntry('inf', 'successfully put '+task+'back in the task queue'))
def returnTask(messageQueue, taskQueue, task):
'''If a task fails and needs to be returned, call this method, it returns the task to the taskQueue
Args:
- messageQueue -- queue used for logging
- taskQueue -- the global task queue from
- task -- the task that should be returned
'''
taskQueue.put(task)
messageQueue.put(UtilityFunctions.createLogEntry('inf', 'successfully put '+task+'back in the task queue'))
def loadResult(resultsDirectory):
'''Zips all data in the results directory in a zip file and returns that zip file loaded into memory. Also removes resultsData from disk '''
temporaryDirectory = './client/temp'
UtilityFunctions.zipDir(resultsDirectory,
os.path.join(temporaryDirectory, 'result.zip'))
resultHandle = open(os.path.join(temporaryDirectory, 'result.zip'), 'r')
resultData = resultHandle.readlines()
resultHandle.close()
UtilityFunctions.deleteContentsOfDirectory(temporaryDirectory)
UtilityFunctions.deleteContentsOfDirectory(resultsDirectory)
return resultData
def loadResult(resultsDirectory):
'''Zips all data in the results directory in a zip file and returns that zip file loaded into memory. Also removes resultsData from disk '''
temporaryDirectory = './client/temp'
UtilityFunctions.zipDir(resultsDirectory, os.path.join(temporaryDirectory,'result.zip'))
resultHandle = open(os.path.join(temporaryDirectory,'result.zip'),'r')
resultData = resultHandle.readlines()
resultHandle.close()
UtilityFunctions.deleteContentsOfDirectory(temporaryDirectory)
UtilityFunctions.deleteContentsOfDirectory(resultsDirectory)
return resultData
def HandleConnection(connection, clientAdress, messageQueue, taskQueue, IOLock,
numberOfConnections, numberOfConnectionsLock,
resultsDirectory):
'''Handles the connection and communication between the server and an individual client
Basic communication process:
1. EDOClient sends a 'requestTask' type message
2. EDOServer asks ServerSideTaskHandler for a Task, ServerSideTaskHandler returns a task.py and a data.zip file that have been loaded into memory. It also returns a string that identifies the task
3. EDOServer sends message to client
4. EDOServer waits for reply
5. EDOClient sends back a 'taskCompleted' type message,
6. EDOServer calls ServerSideTaskHandler.taskCompleted with the received resultComputed Message
7. EDOServer continues at step 2
If the connection to the client dies, EDOServer calls ServerSideTaskHandler.returnTask(taskid)
Args
- connection -- connection to an individual client
- clientAdress -- the address of the client
- messageQueue -- the Queue from which the logger reads
- taskQueue -- the Queue where all the tasks are located
- IOLock --
- numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
- numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
'''
#update number of connections
numberOfConnectionsLock.acquire()
numberOfConnections.value = numberOfConnections.value + 1
numberOfConnectionsLock.release()
task = None
try:
# part of the following while loops was adapted from http://stackoverflow.com/questions/1708835/receving-socket-python
running = True
length = None #parameter telling us the length of the incoming message. Each message is message has the preamble length followed by : followed by data of length length
messageBuffer = "" # a buffer where data is stored until we've received the entire message
data = "" # the received data from the socket
while running == True:
data = connection.recv(1024)
if not data:
raise Exception("Client " + str(clientAdress) +
"sent an empty string, the connection is dead")
messageBuffer += data #add what we received to the buffer
while True:
if length is None: #if length undefined, we must wait until we've received length, we know we have once we see :
if ':' not in messageBuffer:
break
# remove the length bytes from the front of messageBuffer
# leave any remaining bytes in the messageBuffer!
length_str, ignored, messageBuffer = messageBuffer.partition(
':')
length = int(length_str)
if len(messageBuffer) < length:
break
# split off the full message from the remaining bytes
# leave any remaining bytes in the messageBuffer!
message = messageBuffer[:length]
messageBuffer = messageBuffer[length:]
length = None
# PROCESS MESSAGE HERE
receivedMessage = cPickle.loads(message)
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf',
'Received a ' + str(receivedMessage['messageType']) +
'message from client: ' + str(clientAdress)))
if receivedMessage['messageType'] == 'requestTask':
task, clientScript, data = ServerSideTaskHandler.getTask(
messageQueue, taskQueue, IOLock)
if task == None:
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Client ' + str(clientAdress) +
' requested a task, but all tasks are done'))
running = False
break
elif receivedMessage['messageType'] == 'result':
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Client ' + str(clientAdress) +
' completed his task'))
pathToReceivedResult = ServerSideTaskHandler.completeTask(
messageQueue, IOLock, task, receivedMessage['data'],
resultsDirectory)
task = None
# ServerSideResultsProcessor.processResult(messageQueue, IOLock, pathToReceivedResult)
task, clientScript, data = ServerSideTaskHandler.getTask(
messageQueue, taskQueue, IOLock)
if task == None:
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Client ' + str(clientAdress) +
' requested a task, but all tasks are done'))
running = False
break
else:
raise Exception('Received unknown message type' +
receivedMessage['messageType'])
message = UtilityFunctions.createMessage(
'task', clientScript, data)
UtilityFunctions.sendAll(connection, message)
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf',
'sent ' + str(task) + 'to ' + str(clientAdress)))
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Done sending tasks to ' + str(clientAdress)))
except Exception as e:
messageQueue.put(
UtilityFunctions.createLogEntry(
'err', 'EVOServer: Connection: to ' + str(clientAdress) +
'failed because: ' + str(e) + format_exc()))
if not task == None:
ServerSideTaskHandler.returnTask(messageQueue, taskQueue, task)
finally:
numberOfConnectionsLock.acquire()
numberOfConnections.value = numberOfConnections.value - 1
numberOfConnectionsLock.release()
def cleanUp():
'''Message to be called when server exits successfully, cleans up temporary Files'''
UtilityFunctions.deleteContentsOfDirectory('./temp')
def clientMain(port,
resultsName,
pathToTasks,
pathToPreviouslyCompletedTasks=None):
'''Starts the server.
Args:
- port -- port on which the server should listen
- resultsName -- name to which to append the currentTime in order to create a unique string
- pathToTasks -- path to the tasks file
- pathToPreviouslyCompletedTasks -- path to a file containing tasks that have already been completed and should not be executed again.
'''
#-------------initialization of values
timeName = resultsName + datetime.datetime.now().strftime(
'%d%b%Y%H%M%S.%f'
) # a unique id created from the time when the script starts
messageQueue = multiprocessing.Queue(
) # the queue where all log messages are written to.
taskQueue = multiprocessing.Queue(
) # the queue where all tasks that have been loaded into memory and have not been completed are kept
logControlValue = multiprocessing.Value(
'i', int(True)
) #A boolean indicating the the logging process should continue to run. There is no type for bool, so we use int
numberOfConnectionsValue = multiprocessing.Value(
'i', 0
) #An integer counting the number of clients that are currently connected to the server
numberOfConnectionsLock = multiprocessing.Lock(
) #A Lock that should be acquired when modifying the numberOfConnections value
IOLock = multiprocessing.Lock(
) #A lock that should be acquired when performing writing of results to memory
signal.signal(
signal.SIGINT, signal_handler
) #register our signal_handler so that we can detect ctrl-C signals
resultsDirectory, logsDirectory = UtilityFunctions.initializeServerFolderStructure(
timeName
) #directories where results and logs should be stored respectively
if not ServerSideTaskHandler.initializeTaskQueue(
taskQueue, pathToTasks, resultsDirectory,
pathToPreviouslyCompletedTasks):
sys.exit(
1
) # initialize Task Queue returns false if initialization and loading of tasks failed
#-------------done initializing
#-------------start logging module
loggingProcess = multiprocessing.Process(target=LoggingModule.log,
args=(logControlValue,
messageQueue,
logsDirectory))
loggingProcess.start()
#Start listening for clients:
#ListenForClients(port,messageQueue,taskHandler) #DEBUG
listeningProcess = multiprocessing.Process(
target=ListenForClients,
args=(port, messageQueue, taskQueue, IOLock, numberOfConnectionsValue,
numberOfConnectionsLock, resultsDirectory))
listeningProcess.start()
#idle while there are clients connected and tasks still incomplete
now = datetime.datetime.now()
logPeriod = datetime.timedelta(minutes=10)
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Periodic log entry, there are currently:\t ' +
str(taskQueue.qsize()) + ' tasks yet to complete'))
while numberOfConnectionsValue.value > 0 or taskQueue.qsize() > 0:
if datetime.datetime.now() > now + logPeriod:
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Periodic log entry, there are currently:\t ' +
str(taskQueue.qsize()) + ' tasks yet to complete'))
now = datetime.datetime.now()
time.sleep(1)
messageQueue.put(
UtilityFunctions.createLogEntry('inf', 'Completed all tasks, exiting'))
listeningProcess.terminate()
logControlValue.value = int(False)
loggingProcess.join()
listeningProcess.join()
sys.exit(0)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('', port))
s.listen(1)
messageQueue.put(
UtilityFunctions.createLogEntry('inf', 'Listening for clients'))
except socket.error, msg:
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'Bind failed. Error Code : ' + str(msg[0]) +
' Message ' + msg[1] + format_exc()))
raise
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf',
'EVOServer: Listener: Accepting connections on port ' + str(port)))
#accept incoming clients, start a new multiprocessing.Process for each incoming connection
while True:
try:
conn, address = s.accept()
messageQueue.put(
UtilityFunctions.createLogEntry(
'inf', 'New connection from ' + str(address)))
#HandleConnection(conn,address,messageQueue,taskHandler) # DEBUG
multiprocessing.Process(
target=HandleConnection,
args=(conn, address, messageQueue, taskQueue, IOLock,
numberOfConnections, numberOfConnectionsLock,
resultsDirectory)).start()
def cleanUp():
'''Message to be called when server exits successfully, cleans up temporary Files'''
UtilityFunctions.deleteContentsOfDirectory('./temp')
- numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
'''
#create a socket
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('',port))
s.listen(1)
messageQueue.put(UtilityFunctions.createLogEntry('inf', 'Listening for clients'))
except socket.error, msg:
print 'foo'
messageQueue.put(UtilityFunctions.createLogEntry('inf','Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1] + format_exc()))
raise
messageQueue.put(UtilityFunctions.createLogEntry('inf','EVOServer: Listener: Accepting connections on port ' + str(port)))
#accept incoming clients, start a new multiprocessing.Process for each incoming connection
while True:
try:
conn, address = s.accept()
messageQueue.put(UtilityFunctions.createLogEntry('inf','New connection from ' + str(address)))
#HandleConnection(conn,address,messageQueue,taskHandler) # DEBUG
multiprocessing.Process(target=HandleConnection, args=(conn, address, messageQueue, taskQueue, IOLock, numberOfConnections, numberOfConnectionsLock,resultsDirectory)).start()
except socket.error, msg:
messageQueue.put(UtilityFunctions.createLogEntry('inf','Accept failed due to : ' + str(msg[0]) + ' Message ' + msg[1] + format_exc()))
def HandleConnection(connection, clientAdress, messageQueue, taskQueue, IOLock, numberOfConnections, numberOfConnectionsLock, resultsDirectory):
'''Handles the connection and communication between the server and an individual client
def HandleConnection(connection, clientAdress, messageQueue, taskQueue, IOLock, numberOfConnections, numberOfConnectionsLock, resultsDirectory):
'''Handles the connection and communication between the server and an individual client
Basic communication process:
1. EDOClient sends a 'requestTask' type message
2. EDOServer asks ServerSideTaskHandler for a Task, ServerSideTaskHandler returns a task.py and a data.zip file that have been loaded into memory. It also returns a string that identifies the task
3. EDOServer sends message to client
4. EDOServer waits for reply
5. EDOClient sends back a 'taskCompleted' type message,
6. EDOServer calls ServerSideTaskHandler.taskCompleted with the received resultComputed Message
7. EDOServer continues at step 2
If the connection to the client dies, EDOServer calls ServerSideTaskHandler.returnTask(taskid)
Args
- connection -- connection to an individual client
- clientAdress -- the address of the client
- messageQueue -- the Queue from which the logger reads
- taskQueue -- the Queue where all the tasks are located
- IOLock --
- numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
- numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
'''
#update number of connections
numberOfConnectionsLock.acquire()
numberOfConnections.value = numberOfConnections.value+1
numberOfConnectionsLock.release()
task = None
try:
# part of the following while loops was adapted from http://stackoverflow.com/questions/1708835/receving-socket-python
running = True
length = None #parameter telling us the length of the incoming message. Each message is message has the preamble length followed by : followed by data of length length
messageBuffer = "" # a buffer where data is stored until we've received the entire message
data = "" # the received data from the socket
while running == True:
data = connection.recv(1024)
if not data:
raise Exception("Client "+str(clientAdress)+"sent an empty string, the connection is dead")
messageBuffer += data #add what we received to the buffer
while True:
if length is None: #if length undefined, we must wait until we've received length, we know we have once we see :
if ':' not in messageBuffer:
break
# remove the length bytes from the front of messageBuffer
# leave any remaining bytes in the messageBuffer!
length_str, ignored, messageBuffer = messageBuffer.partition(':')
length = int(length_str)
if len(messageBuffer) < length:
break
# split off the full message from the remaining bytes
# leave any remaining bytes in the messageBuffer!
message = messageBuffer[:length]
messageBuffer = messageBuffer[length:]
length = None
# PROCESS MESSAGE HERE
receivedMessage = cPickle.loads(message)
messageQueue.put(UtilityFunctions.createLogEntry('inf','Received a ' + str(receivedMessage['messageType']) + 'message from client: ' + str(clientAdress)))
if receivedMessage['messageType'] == 'requestTask':
task, clientScript,data = ServerSideTaskHandler.getTask(messageQueue,taskQueue,IOLock)
if task == None:
messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' requested a task, but all tasks are done'))
running = False
break;
elif receivedMessage['messageType'] == 'result':
messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' completed his task'))
pathToReceivedResult = ServerSideTaskHandler.completeTask(messageQueue,IOLock, task, receivedMessage['data'], resultsDirectory)
task = None
ServerSideResultsProcessor.processResult(messageQueue, IOLock, pathToReceivedResult)
task, clientScript,data = ServerSideTaskHandler.getTask(messageQueue,taskQueue,IOLock)
if task == None:
messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' requested a task, but all tasks are done'))
running = False
break;
else:
raise Exception('Received unknown message type'+receivedMessage['messageType'])
message = UtilityFunctions.createMessage('task', clientScript, data)
UtilityFunctions.sendAll(connection,message)
messageQueue.put(UtilityFunctions.createLogEntry('inf','sent ' + str(task) + 'to ' + str(clientAdress)))
messageQueue.put(UtilityFunctions.createLogEntry('inf','Done sending tasks to ' + str(clientAdress)))
except Exception as e:
messageQueue.put(UtilityFunctions.createLogEntry('err','EVOServer: Connection: to ' + str(clientAdress) + 'failed because: ' + str(e) + format_exc()))
if not task == None:
ServerSideTaskHandler.returnTask(messageQueue,taskQueue,task)
finally:
numberOfConnectionsLock.acquire()
numberOfConnections.value = numberOfConnections.value-1
numberOfConnectionsLock.release()
def handleConnection(messageQueue, server_address,port,resultsDirectory):
'''Handles Connection to Server. Passes on messages to the ClientSideTaskHandler'''
while True:
try:
connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
connection.connect((server_address,port))
break
except Exception as e:
messageQueue.put(UtilityFunctions.createLogEntry('err','Unable to connect to PyComputeServer, received the following error: ' + str(e)))
time.sleep(20)
messageQueue.put(UtilityFunctions.createLogEntry('inf','Connected to PyComputeServer. ' + server_address +
':' + str(port)))
try:
#tell the server we are ready to process tasks
requestTaskMessage = UtilityFunctions.createMessage('requestTask')
UtilityFunctions.sendAll(connection,requestTaskMessage)
#begin to process tasks sent by the server
running = True
length = None
messageBuffer = ""
data = ""
while running == True:
data = connection.recv(1024)
if not data:
messageQueue.put(UtilityFunctions.createLogEntry('err',"Server sent an empty string, the connection is dead"))
break
messageBuffer += data
#the following loop allows for making sure the entire receivedMessage arrives
while True:
if length is None:
if ':' not in messageBuffer:
break
# remove the length bytes from the front of messageBuffer
# leave any remaining bytes in the messageBuffer!
length_str, ignored, messageBuffer = messageBuffer.partition(':')
length = int(length_str)
if len(messageBuffer) < length:
break
# split off the full receivedMessage from the remaining bytes
# leave any remaining bytes in the messageBuffer!
message = messageBuffer[:length]
messageBuffer = messageBuffer[length:]
length = None
# PROCESS MESSAGE HERE
serverMessage = cPickle.loads(message)
#exec('osreturn='+receivedMessage['data'])
#if osreturn == 0:
print 'executing task'
ClientSideTaskHandler.handleClientTask(messageQueue, serverMessage, resultsDirectory)
resultData = ClientSideTaskHandler.loadResult(resultsDirectory) #we expect the client has put the data in the resultsDirectory
returnMessage = UtilityFunctions.createMessage('result', data=resultData)
UtilityFunctions.sendAll(connection,returnMessage)
except Exception as e:
messageQueue.put(UtilityFunctions.createLogEntry('err',"Connection to server failed because: " + str(e) + format_exc()))
