Esempio n. 1
0
    def executeAndCallback(self,
                           toExecute,
                           outputCallback,
                           inputCallback=None):
        ''' Runs a callable in a separate process, marshalling input/output via callback.

        toExecute      - a callable that can be pickled
        outputCallback - a callback taking a file-descriptor and an integer representing
                         the number of bytes that can be read from the descriptor.
        inputCallback  - a callback that takes a file-descriptor and writes a 4-byte
                         integer representing the size of input followed by the input
                         data itself.
        '''
        with self.lock:
            assert self.hasStarted

            if self.actuallyRunOutOfProcess:
                toSend = pickle.dumps(toExecute)

                os.write(self.parentWriteFD, common.longToString(len(toSend)))
                os.write(self.parentWriteFD, toSend)
                if inputCallback is None:
                    os.write(self.parentWriteFD, common.longToString(0))
                else:
                    inputCallback(self.parentWriteFD)

            else:
                if inputCallback is None:
                    self.writeQueue.put(toExecute)
                else:
                    # fire the callback on a separate thread so we can read from
                    # the pipe while its running and prevernt os.write from
                    # hanging.
                    thread = threading.Thread(target=inputCallback,
                                              args=(self.parentWriteFD, ))
                    thread.start()

                    inputSize = common.stringToLong(
                        os.read(self.childReadFD, 4))
                    inputData = os.read(self.childReadFD, inputSize)

                    thread.join()
                    self.writeQueue.put(lambda: toExecute(inputData))

            prefix = os.read(self.parentReadFD, 5)

            assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
            isException = prefix[0] == BYTE_EXCEPTION

            msgSize = common.stringToLong(prefix[1:5])

            if isException:
                pickledException = os.read(self.parentReadFD, msgSize)
                raise pickle.loads(pickledException)
            else:
                outputCallback(self.parentReadFD, msgSize)
Esempio n. 2
0
    def executeAndCallback(self, toExecute, outputCallback, inputCallback=None):
        ''' Runs a callable in a separate process, marshalling input/output via callback.

        toExecute      - a callable that can be pickled
        outputCallback - a callback taking a file-descriptor and an integer representing
                         the number of bytes that can be read from the descriptor.
        inputCallback  - a callback that takes a file-descriptor and writes a 4-byte
                         integer representing the size of input followed by the input
                         data itself.
        '''
        with self.lock:
            assert self.hasStarted

            if self.actuallyRunOutOfProcess:
                toSend = pickle.dumps(toExecute)

                os.write(self.parentWriteFD, common.longToString(len(toSend)))
                os.write(self.parentWriteFD, toSend)
                if inputCallback is None:
                    os.write(self.parentWriteFD, common.longToString(0))
                else:
                    inputCallback(self.parentWriteFD)

            else:
                if inputCallback is None:
                    self.writeQueue.put(toExecute)
                else:
                    # fire the callback on a separate thread so we can read from
                    # the pipe while its running and prevernt os.write from
                    # hanging.
                    thread = threading.Thread(target=inputCallback, args=(self.parentWriteFD,))
                    thread.start()

                    inputSize = common.stringToLong(os.read(self.childReadFD, 4))
                    inputData = os.read(self.childReadFD, inputSize)

                    thread.join()
                    self.writeQueue.put(lambda: toExecute(inputData))

            prefix = os.read(self.parentReadFD, 5)

            assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
            isException = prefix[0] == BYTE_EXCEPTION

            msgSize = common.stringToLong(prefix[1:5])

            if isException:
                pickledException = os.read(self.parentReadFD, msgSize)
                raise pickle.loads(pickledException)
            else:
                outputCallback(self.parentReadFD, msgSize)
Esempio n. 3
0
    def executeChild_(self):
        logging.info("Child started with %s, %s", self.childWriteFD, self.childReadFD)
        self.hasStarted = True
        self.isChild = True

        try:
            while True:
                if self.actuallyRunOutOfProcess:
                    isException, outgoingMessage = self.runOutOfProc()
                else:
                    isException, outgoingMessage = self.runInProc()
                    if not isException and outgoingMessage is None:
                        return

                finalValueToWrite = (
                    (BYTE_EXCEPTION if isException else BYTE_DATA) +
                    common.longToString(len(outgoingMessage)) + outgoingMessage
                    )

                os.write(self.childWriteFD, finalValueToWrite)
        except:
            logging.error("Main OutOfProcessDownloader loop failed: %s", traceback.format_exc())
        finally:
            #bail
            if self.actuallyRunOutOfProcess:
                os._exit(0)
Esempio n. 4
0
    def executeChild_(self):
        logging.debug("Child started with %s, %s", self.childWriteFD,
                      self.childReadFD)
        self.hasStarted = True
        self.isChild = True

        try:
            while True:
                if self.actuallyRunOutOfProcess:
                    isException, outgoingMessage = self.runOutOfProc()
                else:
                    isException, outgoingMessage = self.runInProc()
                    if not isException and outgoingMessage is None:
                        return

                finalValueToWrite = (
                    (BYTE_EXCEPTION if isException else BYTE_DATA) +
                    common.longToString(len(outgoingMessage)) +
                    outgoingMessage)

                os.write(self.childWriteFD, finalValueToWrite)
        except:
            logging.error("Main OutOfProcessDownloader loop failed: %s",
                          traceback.format_exc())
        finally:
            #bail
            if self.actuallyRunOutOfProcess:
                os._exit(0)
Esempio n. 5
0
    def executeAndCallback(self,
                           toExecute,
                           outputCallback,
                           inputCallback=None):
        ''' Runs a callable in a separate process, marshalling input/output via callback.

        toExecute      - a callable that can be pickled
        outputCallback - a callback taking a file-descriptor and an integer representing
                         the number of bytes that can be read from the descriptor.
        inputCallback  - a callback that takes a file-descriptor and writes a 4-byte
                         integer representing the size of input followed by the input
                         data itself.
        '''
        with self.lock:
            assert self.hasStarted

            if self.actuallyRunOutOfProcess:
                toSend = pickle.dumps(toExecute)

                writeAllToFd(self.parentSocket.fileno(),
                             common.longToString(len(toSend)))
                writeAllToFd(self.parentSocket.fileno(), toSend)

                if inputCallback is None:
                    writeAllToFd(self.parentSocket.fileno(),
                                 common.longToString(0))
                else:
                    inputCallback(self.parentSocket.fileno())

            else:
                if inputCallback is None:
                    self.writeQueue.put(toExecute)
                else:
                    # fire the callback on a separate thread so we can read from
                    # the pipe while its running and prevent os.write from
                    # hanging.
                    def callInputCallback():
                        inputCallback(self.parentSocket.fileno())

                    thread = threading.Thread(target=callInputCallback,
                                              args=())
                    thread.start()

                    direct = False
                    try:
                        if toExecute.wantsDirectAccessToInputFileDescriptor:
                            direct = True
                    except AttributeError:
                        pass

                    if direct:

                        def executeFunc():
                            res = toExecute(self.childSocket.fileno())
                            thread.join()
                            return res

                        self.writeQueue.put(executeFunc)
                    else:
                        inputSize = common.stringToLong(
                            readAtLeast(self.childSocket.fileno(), 4))
                        inputData = readAtLeast(self.childSocket.fileno(),
                                                inputSize)

                        thread.join()

                        self.writeQueue.put(lambda: toExecute(inputData))

            prefix = readAtLeast(self.parentSocket.fileno(), 5)

            if len(prefix) != 5:
                #this downloader is dead
                raise IOError("OutOfProcessDownloader died")

            assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
            isException = prefix[0] == BYTE_EXCEPTION

            msgSize = common.stringToLong(prefix[1:5])

            if isException:
                pickledException = readAtLeast(self.parentSocket.fileno(),
                                               msgSize)
                raise pickle.loads(pickledException)
            else:
                outputCallback(self.parentSocket.fileno(), msgSize)