def executeAndCallback(self,
toExecute,
outputCallback,
inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
os.write(self.parentWriteFD, common.longToString(len(toSend)))
os.write(self.parentWriteFD, toSend)
if inputCallback is None:
os.write(self.parentWriteFD, common.longToString(0))
else:
inputCallback(self.parentWriteFD)
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevernt os.write from
# hanging.
thread = threading.Thread(target=inputCallback,
args=(self.parentWriteFD, ))
thread.start()
inputSize = common.stringToLong(
os.read(self.childReadFD, 4))
inputData = os.read(self.childReadFD, inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = os.read(self.parentReadFD, 5)
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = os.read(self.parentReadFD, msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentReadFD, msgSize)
def executeAndCallback(self, toExecute, outputCallback, inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
os.write(self.parentWriteFD, common.longToString(len(toSend)))
os.write(self.parentWriteFD, toSend)
if inputCallback is None:
os.write(self.parentWriteFD, common.longToString(0))
else:
inputCallback(self.parentWriteFD)
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevernt os.write from
# hanging.
thread = threading.Thread(target=inputCallback, args=(self.parentWriteFD,))
thread.start()
inputSize = common.stringToLong(os.read(self.childReadFD, 4))
inputData = os.read(self.childReadFD, inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = os.read(self.parentReadFD, 5)
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = os.read(self.parentReadFD, msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentReadFD, msgSize)
def runOutOfProc(self):
isException = None
outgoingMessage = None
callableSize = common.stringToLong(os.read(self.childReadFD, 4))
msgCallable = os.read(self.childReadFD, callableSize)
inputSize = common.stringToLong(os.read(self.childReadFD, 4))
msgInput = os.read(self.childReadFD, inputSize) if inputSize > 0 else None
t0 = time.time()
callableObj = None
heartbeatLogger = None
try:
callableObj = pickle.loads(msgCallable)
heartbeatLogger = HeartbeatLogger(str(callableObj))
heartbeatLogger.start()
outgoingMessage = callableObj() if msgInput is None else \
callableObj(msgInput)
assert isinstance(outgoingMessage, str), "Callable %s returned %s, not str" % (callableObj, type(outgoingMessage))
isException = False
except Exception as e:
try:
logging.error(
"OutOfProcessDownloader caught exception after %s seconds: %s\n" +
"Task was %s",
time.time() - t0,
traceback.format_exc(),
callableObj
)
except:
logging.error(
"OutOfProcessDownloader failed formatting error: %s",
traceback.format_exc()
)
isException = True
outgoingMessage = pickle.dumps(e)
finally:
if heartbeatLogger:
heartbeatLogger.stop()
return isException, outgoingMessage
def runOutOfProc(self):
isException = None
outgoingMessage = None
callableSize = common.stringToLong(os.read(self.childReadFD, 4))
msgCallable = os.read(self.childReadFD, callableSize)
inputSize = common.stringToLong(os.read(self.childReadFD, 4))
msgInput = os.read(self.childReadFD,
inputSize) if inputSize > 0 else None
t0 = time.time()
callableObj = None
heartbeatLogger = None
try:
callableObj = pickle.loads(msgCallable)
heartbeatLogger = HeartbeatLogger(str(callableObj))
heartbeatLogger.start()
outgoingMessage = callableObj() if msgInput is None else \
callableObj(msgInput)
assert isinstance(outgoingMessage,
str), "Callable %s returned %s, not str" % (
callableObj, type(outgoingMessage))
isException = False
except Exception as e:
try:
logging.error(
"OutOfProcessDownloader caught exception after %s seconds: %s\n"
+ "Task was %s",
time.time() - t0, traceback.format_exc(), callableObj)
except:
logging.error(
"OutOfProcessDownloader failed formatting error: %s",
traceback.format_exc())
isException = True
outgoingMessage = pickle.dumps(e)
finally:
if heartbeatLogger:
heartbeatLogger.stop()
return isException, outgoingMessage
def executeAndCallback(self,
toExecute,
outputCallback,
inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
writeAllToFd(self.parentSocket.fileno(),
common.longToString(len(toSend)))
writeAllToFd(self.parentSocket.fileno(), toSend)
if inputCallback is None:
writeAllToFd(self.parentSocket.fileno(),
common.longToString(0))
else:
inputCallback(self.parentSocket.fileno())
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevent os.write from
# hanging.
def callInputCallback():
inputCallback(self.parentSocket.fileno())
thread = threading.Thread(target=callInputCallback,
args=())
thread.start()
direct = False
try:
if toExecute.wantsDirectAccessToInputFileDescriptor:
direct = True
except AttributeError:
pass
if direct:
def executeFunc():
res = toExecute(self.childSocket.fileno())
thread.join()
return res
self.writeQueue.put(executeFunc)
else:
inputSize = common.stringToLong(
readAtLeast(self.childSocket.fileno(), 4))
inputData = readAtLeast(self.childSocket.fileno(),
inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = readAtLeast(self.parentSocket.fileno(), 5)
if len(prefix) != 5:
#this downloader is dead
raise IOError("OutOfProcessDownloader died")
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = readAtLeast(self.parentSocket.fileno(),
msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentSocket.fileno(), msgSize)
def runOutOfProc(self):
isException = None
outgoingMessage = None
callableSize = common.stringToLong(
readAtLeast(self.childSocket.fileno(), 4))
msgCallable = readAtLeast(self.childSocket.fileno(), callableSize)
try:
callableObj = pickle.loads(msgCallable)
except Exception as e:
logging.error(
"OutOfProcessDownloader failed deserializing the given callable (of size %s): %s",
callableSize, traceback.format_exc())
isException = True
outgoingMessage = pickle.dumps(e)
return isException, outgoingMessage
direct = False
try:
if callableObj.wantsDirectAccessToInputFileDescriptor:
direct = True
except AttributeError:
pass
if not direct:
inputSize = common.stringToLong(
readAtLeast(self.childSocket.fileno(), 4))
msgInput = readAtLeast(self.childSocket.fileno(),
inputSize) if inputSize > 0 else None
else:
msgInput = self.childSocket.fileno()
t0 = time.time()
heartbeatLogger = None
try:
heartbeatLogger = HeartbeatLogger(str(callableObj))
heartbeatLogger.start()
outgoingMessage = callableObj(
) if msgInput is None else callableObj(msgInput)
assert isinstance(outgoingMessage,
str), "Callable %s returned %s, not str" % (
callableObj, type(outgoingMessage))
isException = False
except Exception as e:
try:
logging.error(
"OutOfProcessDownloader caught exception after %s seconds: %s\n"
+ "Task was %s",
time.time() - t0, traceback.format_exc(), callableObj)
except:
logging.error(
"OutOfProcessDownloader failed formatting error: %s",
traceback.format_exc())
isException = True
outgoingMessage = pickle.dumps(e)
finally:
if heartbeatLogger:
heartbeatLogger.stop()
return isException, outgoingMessage