def executeAndCallback(self, toExecute, outputCallback, inputCallback=None): ''' Runs a callable in a separate process, marshalling input/output via callback. toExecute - a callable that can be pickled outputCallback - a callback taking a file-descriptor and an integer representing the number of bytes that can be read from the descriptor. inputCallback - a callback that takes a file-descriptor and writes a 4-byte integer representing the size of input followed by the input data itself. ''' with self.lock: assert self.hasStarted if self.actuallyRunOutOfProcess: toSend = pickle.dumps(toExecute) os.write(self.parentWriteFD, common.longToString(len(toSend))) os.write(self.parentWriteFD, toSend) if inputCallback is None: os.write(self.parentWriteFD, common.longToString(0)) else: inputCallback(self.parentWriteFD) else: if inputCallback is None: self.writeQueue.put(toExecute) else: # fire the callback on a separate thread so we can read from # the pipe while its running and prevernt os.write from # hanging. thread = threading.Thread(target=inputCallback, args=(self.parentWriteFD, )) thread.start() inputSize = common.stringToLong( os.read(self.childReadFD, 4)) inputData = os.read(self.childReadFD, inputSize) thread.join() self.writeQueue.put(lambda: toExecute(inputData)) prefix = os.read(self.parentReadFD, 5) assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix isException = prefix[0] == BYTE_EXCEPTION msgSize = common.stringToLong(prefix[1:5]) if isException: pickledException = os.read(self.parentReadFD, msgSize) raise pickle.loads(pickledException) else: outputCallback(self.parentReadFD, msgSize)
def executeAndCallback(self, toExecute, outputCallback, inputCallback=None): ''' Runs a callable in a separate process, marshalling input/output via callback. toExecute - a callable that can be pickled outputCallback - a callback taking a file-descriptor and an integer representing the number of bytes that can be read from the descriptor. inputCallback - a callback that takes a file-descriptor and writes a 4-byte integer representing the size of input followed by the input data itself. ''' with self.lock: assert self.hasStarted if self.actuallyRunOutOfProcess: toSend = pickle.dumps(toExecute) os.write(self.parentWriteFD, common.longToString(len(toSend))) os.write(self.parentWriteFD, toSend) if inputCallback is None: os.write(self.parentWriteFD, common.longToString(0)) else: inputCallback(self.parentWriteFD) else: if inputCallback is None: self.writeQueue.put(toExecute) else: # fire the callback on a separate thread so we can read from # the pipe while its running and prevernt os.write from # hanging. thread = threading.Thread(target=inputCallback, args=(self.parentWriteFD,)) thread.start() inputSize = common.stringToLong(os.read(self.childReadFD, 4)) inputData = os.read(self.childReadFD, inputSize) thread.join() self.writeQueue.put(lambda: toExecute(inputData)) prefix = os.read(self.parentReadFD, 5) assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix isException = prefix[0] == BYTE_EXCEPTION msgSize = common.stringToLong(prefix[1:5]) if isException: pickledException = os.read(self.parentReadFD, msgSize) raise pickle.loads(pickledException) else: outputCallback(self.parentReadFD, msgSize)
def runOutOfProc(self): isException = None outgoingMessage = None callableSize = common.stringToLong(os.read(self.childReadFD, 4)) msgCallable = os.read(self.childReadFD, callableSize) inputSize = common.stringToLong(os.read(self.childReadFD, 4)) msgInput = os.read(self.childReadFD, inputSize) if inputSize > 0 else None t0 = time.time() callableObj = None heartbeatLogger = None try: callableObj = pickle.loads(msgCallable) heartbeatLogger = HeartbeatLogger(str(callableObj)) heartbeatLogger.start() outgoingMessage = callableObj() if msgInput is None else \ callableObj(msgInput) assert isinstance(outgoingMessage, str), "Callable %s returned %s, not str" % (callableObj, type(outgoingMessage)) isException = False except Exception as e: try: logging.error( "OutOfProcessDownloader caught exception after %s seconds: %s\n" + "Task was %s", time.time() - t0, traceback.format_exc(), callableObj ) except: logging.error( "OutOfProcessDownloader failed formatting error: %s", traceback.format_exc() ) isException = True outgoingMessage = pickle.dumps(e) finally: if heartbeatLogger: heartbeatLogger.stop() return isException, outgoingMessage
def runOutOfProc(self): isException = None outgoingMessage = None callableSize = common.stringToLong(os.read(self.childReadFD, 4)) msgCallable = os.read(self.childReadFD, callableSize) inputSize = common.stringToLong(os.read(self.childReadFD, 4)) msgInput = os.read(self.childReadFD, inputSize) if inputSize > 0 else None t0 = time.time() callableObj = None heartbeatLogger = None try: callableObj = pickle.loads(msgCallable) heartbeatLogger = HeartbeatLogger(str(callableObj)) heartbeatLogger.start() outgoingMessage = callableObj() if msgInput is None else \ callableObj(msgInput) assert isinstance(outgoingMessage, str), "Callable %s returned %s, not str" % ( callableObj, type(outgoingMessage)) isException = False except Exception as e: try: logging.error( "OutOfProcessDownloader caught exception after %s seconds: %s\n" + "Task was %s", time.time() - t0, traceback.format_exc(), callableObj) except: logging.error( "OutOfProcessDownloader failed formatting error: %s", traceback.format_exc()) isException = True outgoingMessage = pickle.dumps(e) finally: if heartbeatLogger: heartbeatLogger.stop() return isException, outgoingMessage
def executeAndCallback(self, toExecute, outputCallback, inputCallback=None): ''' Runs a callable in a separate process, marshalling input/output via callback. toExecute - a callable that can be pickled outputCallback - a callback taking a file-descriptor and an integer representing the number of bytes that can be read from the descriptor. inputCallback - a callback that takes a file-descriptor and writes a 4-byte integer representing the size of input followed by the input data itself. ''' with self.lock: assert self.hasStarted if self.actuallyRunOutOfProcess: toSend = pickle.dumps(toExecute) writeAllToFd(self.parentSocket.fileno(), common.longToString(len(toSend))) writeAllToFd(self.parentSocket.fileno(), toSend) if inputCallback is None: writeAllToFd(self.parentSocket.fileno(), common.longToString(0)) else: inputCallback(self.parentSocket.fileno()) else: if inputCallback is None: self.writeQueue.put(toExecute) else: # fire the callback on a separate thread so we can read from # the pipe while its running and prevent os.write from # hanging. def callInputCallback(): inputCallback(self.parentSocket.fileno()) thread = threading.Thread(target=callInputCallback, args=()) thread.start() direct = False try: if toExecute.wantsDirectAccessToInputFileDescriptor: direct = True except AttributeError: pass if direct: def executeFunc(): res = toExecute(self.childSocket.fileno()) thread.join() return res self.writeQueue.put(executeFunc) else: inputSize = common.stringToLong( readAtLeast(self.childSocket.fileno(), 4)) inputData = readAtLeast(self.childSocket.fileno(), inputSize) thread.join() self.writeQueue.put(lambda: toExecute(inputData)) prefix = readAtLeast(self.parentSocket.fileno(), 5) if len(prefix) != 5: #this downloader is dead raise IOError("OutOfProcessDownloader died") assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix isException = prefix[0] == BYTE_EXCEPTION msgSize = common.stringToLong(prefix[1:5]) if isException: pickledException = readAtLeast(self.parentSocket.fileno(), msgSize) raise pickle.loads(pickledException) else: outputCallback(self.parentSocket.fileno(), msgSize)
def runOutOfProc(self): isException = None outgoingMessage = None callableSize = common.stringToLong( readAtLeast(self.childSocket.fileno(), 4)) msgCallable = readAtLeast(self.childSocket.fileno(), callableSize) try: callableObj = pickle.loads(msgCallable) except Exception as e: logging.error( "OutOfProcessDownloader failed deserializing the given callable (of size %s): %s", callableSize, traceback.format_exc()) isException = True outgoingMessage = pickle.dumps(e) return isException, outgoingMessage direct = False try: if callableObj.wantsDirectAccessToInputFileDescriptor: direct = True except AttributeError: pass if not direct: inputSize = common.stringToLong( readAtLeast(self.childSocket.fileno(), 4)) msgInput = readAtLeast(self.childSocket.fileno(), inputSize) if inputSize > 0 else None else: msgInput = self.childSocket.fileno() t0 = time.time() heartbeatLogger = None try: heartbeatLogger = HeartbeatLogger(str(callableObj)) heartbeatLogger.start() outgoingMessage = callableObj( ) if msgInput is None else callableObj(msgInput) assert isinstance(outgoingMessage, str), "Callable %s returned %s, not str" % ( callableObj, type(outgoingMessage)) isException = False except Exception as e: try: logging.error( "OutOfProcessDownloader caught exception after %s seconds: %s\n" + "Task was %s", time.time() - t0, traceback.format_exc(), callableObj) except: logging.error( "OutOfProcessDownloader failed formatting error: %s", traceback.format_exc()) isException = True outgoingMessage = pickle.dumps(e) finally: if heartbeatLogger: heartbeatLogger.stop() return isException, outgoingMessage