Example #1
0
def handleConnection(messageQueue, server_address,port,resultsDirectory):
    '''Handles Connection to Server. Passes on messages to the ClientSideTaskHandler'''

    while True:
        try:
            connection = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
            connection.connect((server_address,port))
            break
        
        except Exception as e:
            messageQueue.put(UtilityFunctions.createLogEntry('err','Unable to connect to PyComputeServer, received the following error: ' + str(e)))
            time.sleep(20)
            
        
    
    messageQueue.put(UtilityFunctions.createLogEntry('inf','Connected to PyComputeServer. ' + server_address + 
            ':' + str(port)))
            
    
    try:
        #tell the server we are ready to process tasks
        requestTaskMessage = UtilityFunctions.createMessage('requestTask')
        UtilityFunctions.sendAll(connection,requestTaskMessage)
        #begin to process tasks sent by the server
        running = True
        length = None
        messageBuffer = ""
        data = ""        
        while running == True:
            data = connection.recv(1024)
            if not data:
                messageQueue.put(UtilityFunctions.createLogEntry('err',"Server sent an empty string, the connection is dead"))
                break
            messageBuffer += data
            #the following loop allows for making sure the entire receivedMessage arrives
            while True:
                if length is None:
                    if ':' not in messageBuffer:
                        break
                    # remove the length bytes from the front of messageBuffer
                    # leave any remaining bytes in the messageBuffer!
                    length_str, ignored, messageBuffer = messageBuffer.partition(':')
                    length = int(length_str)
        
                if len(messageBuffer) < length:
                    break
                # split off the full receivedMessage from the remaining bytes
                # leave any remaining bytes in the messageBuffer!
                message = messageBuffer[:length]
                messageBuffer = messageBuffer[length:]
                length = None
                # PROCESS MESSAGE HERE
                serverMessage = cPickle.loads(message)
                
                #exec('osreturn='+receivedMessage['data'])
                #if osreturn == 0:
                print 'executing task'
                ClientSideTaskHandler.handleClientTask(messageQueue, serverMessage, resultsDirectory)
                resultData = ClientSideTaskHandler.loadResult(resultsDirectory) #we expect the client has put the data in the resultsDirectory
                returnMessage = UtilityFunctions.createMessage('result', data=resultData)
                UtilityFunctions.sendAll(connection,returnMessage)
    
    except Exception as e:
        messageQueue.put(UtilityFunctions.createLogEntry('err',"Connection to server failed because: " + str(e) + format_exc()))
Example #2
0
def HandleConnection(connection, clientAdress, messageQueue, taskQueue, IOLock,
                     numberOfConnections, numberOfConnectionsLock,
                     resultsDirectory):
    '''Handles the connection and communication between the server and an individual client
    
    Basic communication process:
        1. EDOClient sends a 'requestTask' type message
        2. EDOServer asks ServerSideTaskHandler for a Task, ServerSideTaskHandler returns a task.py and a data.zip file that have been loaded into memory. It also returns a string that identifies the task
        3. EDOServer sends message to client
        4. EDOServer waits for reply
        5. EDOClient sends back a 'taskCompleted' type message,
        6. EDOServer calls ServerSideTaskHandler.taskCompleted with the received resultComputed Message
        7. EDOServer continues at step 2
    
    If the connection to the client dies, EDOServer calls ServerSideTaskHandler.returnTask(taskid)

    Args
        - connection -- connection to an individual client
        - clientAdress -- the address of the client
        - messageQueue -- the Queue from which the logger reads
        - taskQueue -- the Queue where all the tasks are located
        - IOLock --
        - numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
        - numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
        
    '''

    #update number of connections
    numberOfConnectionsLock.acquire()
    numberOfConnections.value = numberOfConnections.value + 1
    numberOfConnectionsLock.release()

    task = None

    try:
        # part of the following while loops was adapted from http://stackoverflow.com/questions/1708835/receving-socket-python

        running = True

        length = None  #parameter telling us the length of the incoming message. Each message is message has the preamble length followed by : followed by data of length length
        messageBuffer = ""  # a buffer where data is stored until we've received the entire message
        data = ""  # the received data from the socket
        while running == True:
            data = connection.recv(1024)
            if not data:
                raise Exception("Client " + str(clientAdress) +
                                "sent an empty string, the connection is dead")

            messageBuffer += data  #add what we received to the buffer
            while True:
                if length is None:  #if length undefined, we must wait until we've received length, we know we have once we see :
                    if ':' not in messageBuffer:
                        break
                    # remove the length bytes from the front of messageBuffer
                    # leave any remaining bytes in the messageBuffer!
                    length_str, ignored, messageBuffer = messageBuffer.partition(
                        ':')
                    length = int(length_str)

                if len(messageBuffer) < length:
                    break
                # split off the full message from the remaining bytes
                # leave any remaining bytes in the messageBuffer!
                message = messageBuffer[:length]
                messageBuffer = messageBuffer[length:]
                length = None

                # PROCESS MESSAGE HERE

                receivedMessage = cPickle.loads(message)
                messageQueue.put(
                    UtilityFunctions.createLogEntry(
                        'inf',
                        'Received a ' + str(receivedMessage['messageType']) +
                        'message from client: ' + str(clientAdress)))

                if receivedMessage['messageType'] == 'requestTask':

                    task, clientScript, data = ServerSideTaskHandler.getTask(
                        messageQueue, taskQueue, IOLock)
                    if task == None:
                        messageQueue.put(
                            UtilityFunctions.createLogEntry(
                                'inf', 'Client ' + str(clientAdress) +
                                ' requested a task, but all tasks are done'))
                        running = False
                        break

                elif receivedMessage['messageType'] == 'result':
                    messageQueue.put(
                        UtilityFunctions.createLogEntry(
                            'inf', 'Client ' + str(clientAdress) +
                            ' completed his task'))
                    pathToReceivedResult = ServerSideTaskHandler.completeTask(
                        messageQueue, IOLock, task, receivedMessage['data'],
                        resultsDirectory)
                    task = None
                    # ServerSideResultsProcessor.processResult(messageQueue, IOLock, pathToReceivedResult)

                    task, clientScript, data = ServerSideTaskHandler.getTask(
                        messageQueue, taskQueue, IOLock)
                    if task == None:
                        messageQueue.put(
                            UtilityFunctions.createLogEntry(
                                'inf', 'Client ' + str(clientAdress) +
                                ' requested a task, but all tasks are done'))
                        running = False
                        break

                else:
                    raise Exception('Received unknown message type' +
                                    receivedMessage['messageType'])

                message = UtilityFunctions.createMessage(
                    'task', clientScript, data)
                UtilityFunctions.sendAll(connection, message)
                messageQueue.put(
                    UtilityFunctions.createLogEntry(
                        'inf',
                        'sent ' + str(task) + 'to ' + str(clientAdress)))

        messageQueue.put(
            UtilityFunctions.createLogEntry(
                'inf', 'Done sending tasks to ' + str(clientAdress)))
    except Exception as e:
        messageQueue.put(
            UtilityFunctions.createLogEntry(
                'err', 'EVOServer: Connection: to ' + str(clientAdress) +
                'failed because: ' + str(e) + format_exc()))
        if not task == None:
            ServerSideTaskHandler.returnTask(messageQueue, taskQueue, task)

    finally:
        numberOfConnectionsLock.acquire()
        numberOfConnections.value = numberOfConnections.value - 1
        numberOfConnectionsLock.release()
Example #3
0
def HandleConnection(connection, clientAdress, messageQueue, taskQueue, IOLock, numberOfConnections, numberOfConnectionsLock, resultsDirectory):
    '''Handles the connection and communication between the server and an individual client
    
    Basic communication process:
        1. EDOClient sends a 'requestTask' type message
        2. EDOServer asks ServerSideTaskHandler for a Task, ServerSideTaskHandler returns a task.py and a data.zip file that have been loaded into memory. It also returns a string that identifies the task
        3. EDOServer sends message to client
        4. EDOServer waits for reply
        5. EDOClient sends back a 'taskCompleted' type message,
        6. EDOServer calls ServerSideTaskHandler.taskCompleted with the received resultComputed Message
        7. EDOServer continues at step 2
    
    If the connection to the client dies, EDOServer calls ServerSideTaskHandler.returnTask(taskid)

    Args
        - connection -- connection to an individual client
        - clientAdress -- the address of the client
        - messageQueue -- the Queue from which the logger reads
        - taskQueue -- the Queue where all the tasks are located
        - IOLock --
        - numberOfConnections -- a multiprocessing.value which keeps track of the current number of connected clients
        - numberOfConnectionsLock -- a multiprocessing.IOLock which must be aquired when wishing to modify numberOfConnections
        
    '''
    
    
    #update number of connections
    numberOfConnectionsLock.acquire()
    numberOfConnections.value = numberOfConnections.value+1
    numberOfConnectionsLock.release()


    
    task = None
    
    try:
        # part of the following while loops was adapted from http://stackoverflow.com/questions/1708835/receving-socket-python
        
        running = True
        
        length = None #parameter telling us the length of the incoming message. Each message is message has the preamble length followed by : followed by data of length length
        messageBuffer = "" # a buffer where data is stored until we've received the entire message
        data = "" # the received data from the socket
        while running == True:
            data = connection.recv(1024)
            if not data:
                raise Exception("Client "+str(clientAdress)+"sent an empty string, the connection is dead")

            messageBuffer += data #add what we received to the buffer
            while True:
                if length is None: #if length undefined, we must wait until we've received length, we know we have once we see :
                    if ':' not in messageBuffer:
                        break
                    # remove the length bytes from the front of messageBuffer
                    # leave any remaining bytes in the messageBuffer!
                    length_str, ignored, messageBuffer = messageBuffer.partition(':')
                    length = int(length_str)
        
                if len(messageBuffer) < length:
                    break
                # split off the full message from the remaining bytes
                # leave any remaining bytes in the messageBuffer!
                message = messageBuffer[:length]
                messageBuffer = messageBuffer[length:]
                length = None
                
                
                # PROCESS MESSAGE HERE

                receivedMessage = cPickle.loads(message)
                messageQueue.put(UtilityFunctions.createLogEntry('inf','Received a ' + str(receivedMessage['messageType']) + 'message from client: ' + str(clientAdress)))                            
            

                if receivedMessage['messageType'] == 'requestTask':
                
                    task, clientScript,data = ServerSideTaskHandler.getTask(messageQueue,taskQueue,IOLock)
                    if task == None:
                        messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' requested a task, but all tasks are done'))
                        running = False
                        break;
                    
                elif receivedMessage['messageType'] == 'result':
                    messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' completed his task'))
                    pathToReceivedResult = ServerSideTaskHandler.completeTask(messageQueue,IOLock, task, receivedMessage['data'], resultsDirectory)
                    task = None
                    ServerSideResultsProcessor.processResult(messageQueue, IOLock, pathToReceivedResult)
                    
                    task, clientScript,data = ServerSideTaskHandler.getTask(messageQueue,taskQueue,IOLock)
                    if task == None:
                        messageQueue.put(UtilityFunctions.createLogEntry('inf','Client '+str(clientAdress)+' requested a task, but all tasks are done'))
                        running = False
                        break;
                
                    
                else:
                    raise Exception('Received unknown message type'+receivedMessage['messageType'])
                
                message = UtilityFunctions.createMessage('task', clientScript, data)
                UtilityFunctions.sendAll(connection,message)
                messageQueue.put(UtilityFunctions.createLogEntry('inf','sent ' + str(task) + 'to ' + str(clientAdress)))
        
        messageQueue.put(UtilityFunctions.createLogEntry('inf','Done sending tasks to ' + str(clientAdress)))
    except Exception as e:
        messageQueue.put(UtilityFunctions.createLogEntry('err','EVOServer: Connection: to ' + str(clientAdress) + 'failed because: ' + str(e) + format_exc()))
        if not task == None:
            ServerSideTaskHandler.returnTask(messageQueue,taskQueue,task)
    
    finally:
        numberOfConnectionsLock.acquire()
        numberOfConnections.value = numberOfConnections.value-1
        numberOfConnectionsLock.release()