def executeAndCallback(self,
toExecute,
outputCallback,
inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
os.write(self.parentWriteFD, common.longToString(len(toSend)))
os.write(self.parentWriteFD, toSend)
if inputCallback is None:
os.write(self.parentWriteFD, common.longToString(0))
else:
inputCallback(self.parentWriteFD)
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevernt os.write from
# hanging.
thread = threading.Thread(target=inputCallback,
args=(self.parentWriteFD, ))
thread.start()
inputSize = common.stringToLong(
os.read(self.childReadFD, 4))
inputData = os.read(self.childReadFD, inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = os.read(self.parentReadFD, 5)
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = os.read(self.parentReadFD, msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentReadFD, msgSize)
def executeAndCallback(self, toExecute, outputCallback, inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
os.write(self.parentWriteFD, common.longToString(len(toSend)))
os.write(self.parentWriteFD, toSend)
if inputCallback is None:
os.write(self.parentWriteFD, common.longToString(0))
else:
inputCallback(self.parentWriteFD)
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevernt os.write from
# hanging.
thread = threading.Thread(target=inputCallback, args=(self.parentWriteFD,))
thread.start()
inputSize = common.stringToLong(os.read(self.childReadFD, 4))
inputData = os.read(self.childReadFD, inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = os.read(self.parentReadFD, 5)
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = os.read(self.parentReadFD, msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentReadFD, msgSize)
def executeChild_(self):
logging.info("Child started with %s, %s", self.childWriteFD, self.childReadFD)
self.hasStarted = True
self.isChild = True
try:
while True:
if self.actuallyRunOutOfProcess:
isException, outgoingMessage = self.runOutOfProc()
else:
isException, outgoingMessage = self.runInProc()
if not isException and outgoingMessage is None:
return
finalValueToWrite = (
(BYTE_EXCEPTION if isException else BYTE_DATA) +
common.longToString(len(outgoingMessage)) + outgoingMessage
)
os.write(self.childWriteFD, finalValueToWrite)
except:
logging.error("Main OutOfProcessDownloader loop failed: %s", traceback.format_exc())
finally:
#bail
if self.actuallyRunOutOfProcess:
os._exit(0)
def executeChild_(self):
logging.debug("Child started with %s, %s", self.childWriteFD,
self.childReadFD)
self.hasStarted = True
self.isChild = True
try:
while True:
if self.actuallyRunOutOfProcess:
isException, outgoingMessage = self.runOutOfProc()
else:
isException, outgoingMessage = self.runInProc()
if not isException and outgoingMessage is None:
return
finalValueToWrite = (
(BYTE_EXCEPTION if isException else BYTE_DATA) +
common.longToString(len(outgoingMessage)) +
outgoingMessage)
os.write(self.childWriteFD, finalValueToWrite)
except:
logging.error("Main OutOfProcessDownloader loop failed: %s",
traceback.format_exc())
finally:
#bail
if self.actuallyRunOutOfProcess:
os._exit(0)
def executeAndCallback(self,
toExecute,
outputCallback,
inputCallback=None):
''' Runs a callable in a separate process, marshalling input/output via callback.
toExecute - a callable that can be pickled
outputCallback - a callback taking a file-descriptor and an integer representing
the number of bytes that can be read from the descriptor.
inputCallback - a callback that takes a file-descriptor and writes a 4-byte
integer representing the size of input followed by the input
data itself.
'''
with self.lock:
assert self.hasStarted
if self.actuallyRunOutOfProcess:
toSend = pickle.dumps(toExecute)
writeAllToFd(self.parentSocket.fileno(),
common.longToString(len(toSend)))
writeAllToFd(self.parentSocket.fileno(), toSend)
if inputCallback is None:
writeAllToFd(self.parentSocket.fileno(),
common.longToString(0))
else:
inputCallback(self.parentSocket.fileno())
else:
if inputCallback is None:
self.writeQueue.put(toExecute)
else:
# fire the callback on a separate thread so we can read from
# the pipe while its running and prevent os.write from
# hanging.
def callInputCallback():
inputCallback(self.parentSocket.fileno())
thread = threading.Thread(target=callInputCallback,
args=())
thread.start()
direct = False
try:
if toExecute.wantsDirectAccessToInputFileDescriptor:
direct = True
except AttributeError:
pass
if direct:
def executeFunc():
res = toExecute(self.childSocket.fileno())
thread.join()
return res
self.writeQueue.put(executeFunc)
else:
inputSize = common.stringToLong(
readAtLeast(self.childSocket.fileno(), 4))
inputData = readAtLeast(self.childSocket.fileno(),
inputSize)
thread.join()
self.writeQueue.put(lambda: toExecute(inputData))
prefix = readAtLeast(self.parentSocket.fileno(), 5)
if len(prefix) != 5:
#this downloader is dead
raise IOError("OutOfProcessDownloader died")
assert prefix[0] in (BYTE_EXCEPTION, BYTE_DATA), prefix
isException = prefix[0] == BYTE_EXCEPTION
msgSize = common.stringToLong(prefix[1:5])
if isException:
pickledException = readAtLeast(self.parentSocket.fileno(),
msgSize)
raise pickle.loads(pickledException)
else:
outputCallback(self.parentSocket.fileno(), msgSize)