def listen(self):
stdout = sys.stdout
# Mute stdout. Nobody should actually be able to write to it,
# because stdout is used for IPC.
sys.stdout = open(os.devnull, 'w')
stdin = sys.stdin
if sys.version_info[0] > 2:
stdout = stdout.buffer
stdin = stdin.buffer
# Python 2 opens streams in text mode on Windows. Set stdout and stdin
# to binary mode.
elif sys.platform == 'win32':
import msvcrt
msvcrt.setmode(stdout.fileno(), os.O_BINARY)
msvcrt.setmode(stdin.fileno(), os.O_BINARY)
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed.
# Don't make a big fuss here and just exit.
exit(0)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, stdout, self._pickle_protocol)
def listen(self):
stdout = sys.stdout
# Mute stdout/stderr. Nobody should actually be able to write to those,
# because stdout is used for IPC and stderr will just be annoying if it
# leaks (on module imports).
sys.stdout = open(os.devnull, 'w')
sys.stderr = open(os.devnull, 'w')
stdin = sys.stdin
if sys.version_info[0] > 2:
stdout = stdout.buffer
stdin = stdin.buffer
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed. Don't make a big fuss
# here and just exit.
exit(1)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, file=stdout)
def listen(self):
stdout = sys.stdout
# Mute stdout. Nobody should actually be able to write to it,
# because stdout is used for IPC.
sys.stdout = open(os.devnull, 'w')
stdin = sys.stdin
if sys.version_info[0] > 2:
stdout = stdout.buffer
stdin = stdin.buffer
# Python 2 opens streams in text mode on Windows. Set stdout and stdin
# to binary mode.
elif sys.platform == 'win32':
import msvcrt
msvcrt.setmode(stdout.fileno(), os.O_BINARY)
msvcrt.setmode(stdin.fileno(), os.O_BINARY)
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed.
# Don't make a big fuss here and just exit.
exit(0)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, stdout, self._pickle_protocol)
def listen(self):
stdout = sys.stdout
# Mute stdout/stderr. Nobody should actually be able to write to those,
# because stdout is used for IPC and stderr will just be annoying if it
# leaks (on module imports).
sys.stdout = open(os.devnull, 'w')
sys.stderr = open(os.devnull, 'w')
stdin = sys.stdin
if sys.version_info[0] > 2:
stdout = stdout.buffer
stdin = stdin.buffer
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed. Don't make a big fuss
# here and just exit.
exit(1)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, file=stdout)
def _send(self, evaluator_id, function, args=(), kwargs={}):
if self._crashed:
raise InternalError("The subprocess %s has crashed." % self._executable)
if not is_py3:
# Python 2 compatibility
kwargs = {force_unicode(key): value for key, value in kwargs.items()}
data = evaluator_id, function, args, kwargs
try:
pickle_dump(data, self._process.stdin)
except (socket.error, IOError) as e:
# Once Python2 will be removed we can just use `BrokenPipeError`.
# Also, somehow in windows it returns EINVAL instead of EPIPE if
# the subprocess dies.
if e.errno not in (errno.EPIPE, errno.EINVAL):
# Not a broken pipe
raise
self.kill()
raise InternalError("The subprocess %s was killed. Maybe out of memory?"
% self._executable)
try:
is_exception, traceback, result = pickle_load(self._process.stdout)
except EOFError:
self.kill()
raise InternalError("The subprocess %s has crashed." % self._executable)
if is_exception:
# Replace the attribute error message with a the traceback. It's
# way more informative.
result.args = (traceback,)
raise result
return result
def _send(self, evaluator_id, function, args=(), kwargs={}):
if self.is_crashed:
raise InternalError("The subprocess %s has crashed." %
self._executable)
if not is_py3:
# Python 2 compatibility
kwargs = {
force_unicode(key): value
for key, value in kwargs.items()
}
data = evaluator_id, function, args, kwargs
try:
pickle_dump(data, self._get_process().stdin, self._pickle_protocol)
except (socket.error, IOError) as e:
# Once Python2 will be removed we can just use `BrokenPipeError`.
# Also, somehow in windows it returns EINVAL instead of EPIPE if
# the subprocess dies.
if e.errno not in (errno.EPIPE, errno.EINVAL):
# Not a broken pipe
raise
self._kill()
raise InternalError(
"The subprocess %s was killed. Maybe out of memory?" %
self._executable)
try:
is_exception, traceback, result = pickle_load(
self._get_process().stdout)
except EOFError as eof_error:
try:
stderr = self._get_process().stderr.read().decode(
'utf-8', 'replace')
except Exception as exc:
stderr = '<empty/not available (%r)>' % exc
self._kill()
_add_stderr_to_debug(self._stderr_queue)
raise InternalError(
"The subprocess %s has crashed (%r, stderr=%s)." % (
self._executable,
eof_error,
stderr,
))
_add_stderr_to_debug(self._stderr_queue)
if is_exception:
# Replace the attribute error message with a the traceback. It's
# way more informative.
result.args = (traceback, )
raise result
return result
def _send(self, evaluator_id, function, args=(), kwargs={}):
if self.is_crashed:
raise InternalError("The subprocess %s has crashed." % self._executable)
if not is_py3:
# Python 2 compatibility
kwargs = {force_unicode(key): value for key, value in kwargs.items()}
data = evaluator_id, function, args, kwargs
try:
pickle_dump(data, self._process.stdin, self._pickle_protocol)
except (socket.error, IOError) as e:
# Once Python2 will be removed we can just use `BrokenPipeError`.
# Also, somehow in windows it returns EINVAL instead of EPIPE if
# the subprocess dies.
if e.errno not in (errno.EPIPE, errno.EINVAL):
# Not a broken pipe
raise
self._kill()
raise InternalError("The subprocess %s was killed. Maybe out of memory?"
% self._executable)
try:
is_exception, traceback, result = pickle_load(self._process.stdout)
except EOFError as eof_error:
try:
stderr = self._process.stderr.read().decode('utf-8', 'replace')
except Exception as exc:
stderr = '<empty/not available (%r)>' % exc
self._kill()
_add_stderr_to_debug(self._stderr_queue)
raise InternalError(
"The subprocess %s has crashed (%r, stderr=%s)." % (
self._executable,
eof_error,
stderr,
))
_add_stderr_to_debug(self._stderr_queue)
if is_exception:
# Replace the attribute error message with a the traceback. It's
# way more informative.
result.args = (traceback,)
raise result
return result
def listen(self):
stdout = sys.stdout
stdin = sys.stdin
if sys.version_info[0] > 2:
stdout = stdout.buffer
stdin = stdin.buffer
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed. Don't make a big fuss
# here and just exit.
exit(1)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, file=stdout)
stdout.flush()
def _send(self, inference_state_id, function, args=(), kwargs={}):
if self.is_crashed:
raise InternalError("The subprocess %s has crashed." %
self._executable)
data = inference_state_id, function, args, kwargs
try:
pickle_dump(data, self._get_process().stdin, PICKLE_PROTOCOL)
except BrokenPipeError:
self._kill()
raise InternalError(
"The subprocess %s was killed. Maybe out of memory?" %
self._executable)
try:
is_exception, traceback, result = pickle_load(
self._get_process().stdout)
except EOFError as eof_error:
try:
stderr = self._get_process().stderr.read().decode(
'utf-8', 'replace')
except Exception as exc:
stderr = '<empty/not available (%r)>' % exc
self._kill()
_add_stderr_to_debug(self._stderr_queue)
raise InternalError(
"The subprocess %s has crashed (%r, stderr=%s)." % (
self._executable,
eof_error,
stderr,
))
_add_stderr_to_debug(self._stderr_queue)
if is_exception:
# Replace the attribute error message with a the traceback. It's
# way more informative.
result.args = (traceback, )
raise result
return result
def _send(self, evaluator_id, function, args=(), kwargs={}):
if self._crashed:
raise InternalError("The subprocess %s has crashed." %
self._executable)
if not is_py3:
# Python 2 compatibility
kwargs = {
force_unicode(key): value
for key, value in kwargs.items()
}
data = evaluator_id, function, args, kwargs
try:
pickle_dump(data, self._process.stdin)
except (socket.error, IOError) as e:
# Once Python2 will be removed we can just use `BrokenPipeError`.
# Also, somehow in windows it returns EINVAL instead of EPIPE if
# the subprocess dies.
if e.errno not in (errno.EPIPE, errno.EINVAL):
# Not a broken pipe
raise
self.kill()
raise InternalError(
"The subprocess %s was killed. Maybe out of memory?" %
self._executable)
try:
is_exception, traceback, result = pickle_load(self._process.stdout)
except EOFError:
self.kill()
raise InternalError("The subprocess %s has crashed." %
self._executable)
if is_exception:
# Replace the attribute error message with a the traceback. It's
# way more informative.
result.args = (traceback, )
raise result
return result
def listen(self):
stdout = sys.stdout
# Mute stdout. Nobody should actually be able to write to it,
# because stdout is used for IPC.
sys.stdout = open(os.devnull, 'w')
stdin = sys.stdin
stdout = stdout.buffer
stdin = stdin.buffer
while True:
try:
payload = pickle_load(stdin)
except EOFError:
# It looks like the parent process closed.
# Don't make a big fuss here and just exit.
exit(0)
try:
result = False, None, self._run(*payload)
except Exception as e:
result = True, traceback.format_exc(), e
pickle_dump(result, stdout, PICKLE_PROTOCOL)
