def _download_descriptors(self, retries):
try:
use_authority = retries == 0 and self.fall_back_to_authority
self.download_url = self._pick_url(use_authority)
self.start_time = time.time()
response = urllib2.urlopen(self.download_url, timeout = self.timeout).read()
if self.download_url.endswith('.z'):
response = zlib.decompress(response)
self.content = response.strip()
self.runtime = time.time() - self.start_time
log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
except:
exc = sys.exc_info()[1]
if retries > 0:
log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
return self._download_descriptors(retries - 1)
else:
log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
self.error = exc
finally:
self.is_done = True
def send_message(control_file, message, raw=False):
"""
Sends a message to the control socket, adding the expected formatting for
single verses multi-line messages. Neither message type should contain an
ending newline (if so it'll be treated as a multi-line message with a blank
line at the end). If the message doesn't contain a newline then it's sent
as...
::
<message>\\r\\n
and if it does contain newlines then it's split on ``\\n`` and sent as...
::
+<line 1>\\r\\n
<line 2>\\r\\n
<line 3>\\r\\n
.\\r\\n
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:param str message: message to be sent on the control socket
:param bool raw: leaves the message formatting untouched, passing it to the
socket as-is
:raises:
* :class:`stem.SocketError` if a problem arises in using the socket
* :class:`stem.SocketClosed` if the socket is known to be shut down
"""
if not raw:
message = send_formatting(message)
try:
control_file.write(stem.util.str_tools._to_bytes(message))
control_file.flush()
if log.is_tracing():
log_message = message.replace('\r\n', '\n').rstrip()
msg_div = '\n' if '\n' in log_message else ' '
log.trace('Sent to tor:%s%s' % (msg_div, log_message))
except socket.error as exc:
log.info('Failed to send message: %s' % exc)
# When sending there doesn't seem to be a reliable method for
# distinguishing between failures from a disconnect verses other things.
# Just accounting for known disconnection responses.
if str(exc) == '[Errno 32] Broken pipe':
raise stem.SocketClosed(exc)
else:
raise stem.SocketError(exc)
except AttributeError:
# if the control_file has been closed then flush will receive:
# AttributeError: 'NoneType' object has no attribute 'sendall'
log.info('Failed to send message: file has been closed')
raise stem.SocketClosed('file has been closed')
def _download_descriptors(self, retries):
try:
use_authority = retries == 0 and self.fall_back_to_authority
self.download_url = self._pick_url(use_authority)
self.start_time = time.time()
response = urllib2.urlopen(self.download_url, timeout = self.timeout).read()
if self.download_url.endswith('.z'):
response = zlib.decompress(response)
self.content = response.strip()
self.runtime = time.time() - self.start_time
log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
except:
exc = sys.exc_info()[1]
if retries > 0:
log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
return self._download_descriptors(retries - 1)
else:
log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
self.error = exc
finally:
self.is_done = True
def _download_descriptors(self, retries, timeout):
try:
self.start_time = time.time()
endpoint = self._pick_endpoint(use_authority = retries == 0 and self.fall_back_to_authority)
if isinstance(endpoint, stem.ORPort):
self.content, self.reply_headers = _download_from_orport(endpoint, self.compression, self.resource)
elif isinstance(endpoint, stem.DirPort):
self.download_url = 'http://%s:%i/%s' % (endpoint.address, endpoint.port, self.resource.lstrip('/'))
self.content, self.reply_headers = _download_from_dirport(self.download_url, self.compression, timeout)
else:
raise ValueError("BUG: endpoints can only be ORPorts or DirPorts, '%s' was a %s" % (endpoint, type(endpoint).__name__))
self.runtime = time.time() - self.start_time
log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
except:
exc = sys.exc_info()[1]
if timeout is not None:
timeout -= time.time() - self.start_time
if retries > 0 and (timeout is None or timeout > 0):
log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
return self._download_descriptors(retries - 1, timeout)
else:
log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
self.error = exc
finally:
self.is_done = True
async def _download_descriptors(self, retries: int,
timeout: Optional[float]) -> None:
self.start_time = time.time()
retries = self.retries
time_remaining = self.timeout
while True:
endpoint = self._pick_endpoint(
use_authority=retries == 0 and self.fall_back_to_authority)
if isinstance(endpoint, stem.ORPort):
downloaded_from = 'ORPort %s:%s (resource %s)' % (
endpoint.address, endpoint.port, self.resource)
elif isinstance(endpoint, stem.DirPort):
downloaded_from = 'http://%s:%i/%s' % (
endpoint.address, endpoint.port, self.resource.lstrip('/'))
self.download_url = downloaded_from
else:
raise ValueError(
"BUG: endpoints can only be ORPorts or DirPorts, '%s' was a %s"
% (endpoint, type(endpoint).__name__))
try:
response = await asyncio.wait_for(
self._download_from(endpoint), time_remaining)
self.content, self.reply_headers = _http_body_and_headers(
response)
self.is_done = True
self.runtime = time.time() - self.start_time
log.trace('Descriptors retrieved from %s in %0.2fs' %
(downloaded_from, self.runtime))
return
except asyncio.TimeoutError as exc:
self.is_done = True
self.error = stem.DownloadTimeout(downloaded_from, exc,
sys.exc_info()[2],
self.timeout)
return
except:
exception = sys.exc_info()[1]
retries -= 1
if time_remaining is not None:
time_remaining -= time.time() - self.start_time
if retries > 0:
log.debug(
"Failed to download descriptors from '%s' (%i retries remaining): %s"
% (downloaded_from, retries, exception))
else:
log.debug("Failed to download descriptors from '%s': %s" %
(self.download_url, exception))
self.is_done = True
self.error = exception
return
def send_message(control_file, message, raw = False):
"""
Sends a message to the control socket, adding the expected formatting for
single verses multi-line messages. Neither message type should contain an
ending newline (if so it'll be treated as a multi-line message with a blank
line at the end). If the message doesn't contain a newline then it's sent
as...
::
<message>\\r\\n
and if it does contain newlines then it's split on ``\\n`` and sent as...
::
+<line 1>\\r\\n
<line 2>\\r\\n
<line 3>\\r\\n
.\\r\\n
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:param str message: message to be sent on the control socket
:param bool raw: leaves the message formatting untouched, passing it to the
socket as-is
:raises:
* :class:`stem.SocketError` if a problem arises in using the socket
* :class:`stem.SocketClosed` if the socket is known to be shut down
"""
if not raw:
message = send_formatting(message)
try:
control_file.write(stem.util.str_tools._to_bytes(message))
control_file.flush()
log_message = message.replace('\r\n', '\n').rstrip()
log.trace('Sent to tor:\n' + log_message)
except socket.error as exc:
log.info('Failed to send message: %s' % exc)
# When sending there doesn't seem to be a reliable method for
# distinguishing between failures from a disconnect verses other things.
# Just accounting for known disconnection responses.
if str(exc) == '[Errno 32] Broken pipe':
raise stem.SocketClosed(exc)
else:
raise stem.SocketError(exc)
except AttributeError:
# if the control_file has been closed then flush will receive:
# AttributeError: 'NoneType' object has no attribute 'sendall'
log.info('Failed to send message: file has been closed')
raise stem.SocketClosed('file has been closed')
def send_message(control_file, message, raw=False):
"""
Sends a message to the control socket, adding the expected formatting for
single verses multi-line messages. Neither message type should contain an
ending newline (if so it'll be treated as a multi-line message with a blank
line at the end). If the message doesn't contain a newline then it's sent
as...
::
<message>\\r\\n
and if it does contain newlines then it's split on ``\\n`` and sent as...
::
+<line 1>\\r\\n
<line 2>\\r\\n
<line 3>\\r\\n
.\\r\\n
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:param str message: message to be sent on the control socket
:param bool raw: leaves the message formatting untouched, passing it to the
socket as-is
:raises:
* :class:`stem.SocketError` if a problem arises in using the socket
* :class:`stem.SocketClosed` if the socket is known to be shut down
"""
if not raw:
message = send_formatting(message)
try:
control_file.write(message)
control_file.flush()
log_message = message.replace("\r\n", "\n").rstrip()
log.trace("Sent to tor:\n" + log_message)
except socket.error, exc:
log.info("Failed to send message: %s" % exc)
# When sending there doesn't seem to be a reliable method for
# distinguishing between failures from a disconnect verses other things.
# Just accounting for known disconnection responses.
if str(exc) == "[Errno 32] Broken pipe":
raise stem.SocketClosed(exc)
else:
raise stem.SocketError(exc)
def _log_trace(response):
if not log.is_tracing():
return
log_message = stem.util.str_tools._to_unicode(response.replace(b'\r\n', b'\n').rstrip())
log_message_lines = log_message.split('\n')
if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS:
log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + ['... %i more lines...' % (len(log_message_lines) - TRUNCATE_LOGS)])
if len(log_message_lines) > 2:
log.trace('Received from tor:\n%s' % log_message)
else:
log.trace('Received from tor: %s' % log_message.replace('\n', '\\n'))
async def send_message(writer: asyncio.StreamWriter,
message: Union[bytes, str],
raw: bool = False) -> None:
"""
Sends a message to the control socket, adding the expected formatting for
single verses multi-line messages. Neither message type should contain an
ending newline (if so it'll be treated as a multi-line message with a blank
line at the end). If the message doesn't contain a newline then it's sent
as...
::
<message>\\r\\n
and if it does contain newlines then it's split on ``\\n`` and sent as...
::
+<line 1>\\r\\n
<line 2>\\r\\n
<line 3>\\r\\n
.\\r\\n
:param writer: writer object
:param message: message to be sent on the control socket
:param raw: leaves the message formatting untouched, passing it to the
socket as-is
:raises:
* :class:`stem.SocketError` if a problem arises in using the socket
* :class:`stem.SocketClosed` if the socket is known to be shut down
"""
message = stem.util.str_tools._to_unicode(message)
if not raw:
message = send_formatting(message)
await _write_to_socket(writer, message)
if log.is_tracing():
log_message = message.replace('\r\n', '\n').rstrip()
msg_div = '\n' if '\n' in log_message else ' '
log.trace('Sent to tor:%s%s' % (msg_div, log_message))
def send_message(control_file, message, raw = False):
"""
Sends a message to the control socket, adding the expected formatting for
single verses multi-line messages. Neither message type should contain an
ending newline (if so it'll be treated as a multi-line message with a blank
line at the end). If the message doesn't contain a newline then it's sent
as...
::
<message>\\r\\n
and if it does contain newlines then it's split on ``\\n`` and sent as...
::
+<line 1>\\r\\n
<line 2>\\r\\n
<line 3>\\r\\n
.\\r\\n
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:param str message: message to be sent on the control socket
:param bool raw: leaves the message formatting untouched, passing it to the
socket as-is
:raises:
* :class:`stem.SocketError` if a problem arises in using the socket
* :class:`stem.SocketClosed` if the socket is known to be shut down
"""
if not raw:
message = send_formatting(message)
_write_to_socket(control_file, message)
if log.is_tracing():
log_message = message.replace('\r\n', '\n').rstrip()
msg_div = '\n' if '\n' in log_message else ' '
log.trace('Sent to tor:%s%s' % (msg_div, log_message))
def _setup_debug_logging(args):
"""
Configures us to log at stem's trace level to a debug log path. This starts
it off with some general diagnostic information.
"""
debug_dir = os.path.dirname(args.debug_path)
if not os.path.exists(debug_dir):
os.makedirs(debug_dir)
debug_handler = logging.FileHandler(args.debug_path, mode = 'w')
debug_handler.setLevel(stem.util.log.logging_level(stem.util.log.TRACE))
debug_handler.setFormatter(logging.Formatter(
fmt = '%(asctime)s [%(levelname)s] %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S'
))
logger = stem.util.log.get_logger()
logger.addHandler(debug_handler)
nyxrc_content = "[file doesn't exist]"
if os.path.exists(args.config):
try:
with open(args.config) as nyxrc_file:
nyxrc_content = nyxrc_file.read()
except IOError as exc:
nyxrc_content = '[unable to read file: %s]' % exc.strerror
log.trace(
'debug.header',
nyx_version = nyx.__version__,
stem_version = stem.__version__,
python_version = '.'.join(map(str, sys.version_info[:3])),
system = platform.system(),
platform = ' '.join(platform.dist()),
nyxrc_path = args.config,
nyxrc_content = nyxrc_content,
)
def _setup_debug_logging(args):
"""
Configures us to log at stem's trace level to a debug log path. This starts
it off with some general diagnostic information.
"""
debug_dir = os.path.dirname(args.debug_path)
if not os.path.exists(debug_dir):
os.makedirs(debug_dir)
debug_handler = logging.FileHandler(args.debug_path, mode = 'w')
debug_handler.setLevel(stem.util.log.logging_level(stem.util.log.TRACE))
debug_handler.setFormatter(logging.Formatter(
fmt = '%(asctime)s [%(levelname)s] %(message)s',
datefmt = '%m/%d/%Y %H:%M:%S'
))
logger = stem.util.log.get_logger()
logger.addHandler(debug_handler)
nyxrc_content = "[file doesn't exist]"
if os.path.exists(args.config):
try:
with open(args.config) as nyxrc_file:
nyxrc_content = nyxrc_file.read()
except IOError as exc:
nyxrc_content = '[unable to read file: %s]' % exc.strerror
log.trace(
'debug.header',
nyx_version = nyx.__version__,
stem_version = stem.__version__,
python_version = '.'.join(map(str, sys.version_info[:3])),
system = platform.system(),
platform = ' '.join(platform.dist()),
nyxrc_path = args.config,
nyxrc_content = nyxrc_content,
)
def call(command, default = UNDEFINED, ignore_exit_status = False):
"""
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
:param str,list command: command to be issued
:param object default: response if the query fails
:param bool ignore_exit_status: reports failure if our command's exit status
was non-zero
:returns: **list** with the lines of output from the command
:raises: **OSError** if this fails and no default was provided
"""
if isinstance(command, str):
command_list = command.split(' ')
else:
command_list = command
try:
is_shell_command = command_list[0] in SHELL_COMMANDS
start_time = time.time()
process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command)
stdout, stderr = process.communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
trace_prefix = 'Received from system (%s)' % command
if stdout and stderr:
log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
elif stdout:
log.trace(trace_prefix + ', stdout:\n%s' % stdout)
elif stderr:
log.trace(trace_prefix + ', stderr:\n%s' % stderr)
exit_code = process.poll()
if not ignore_exit_status and exit_code != 0:
raise OSError('%s returned exit status %i' % (command, exit_code))
if stdout:
return stdout.decode('utf-8', 'replace').splitlines()
else:
return []
except OSError as exc:
log.debug('System call (failed): %s (error: %s)' % (command, exc))
if default != UNDEFINED:
return default
else:
raise exc
def call(command, default=UNDEFINED, ignore_exit_status=False):
"""
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
:param str command: command to be issued
:param object default: response if the query fails
:param bool ignore_exit_status: reports failure if our command's exit status
was non-zero
:returns: **list** with the lines of output from the command
:raises: **OSError** if this fails and no default was provided
"""
try:
is_shell_command = command.split(" ")[0] in SHELL_COMMANDS
start_time = time.time()
process = subprocess.Popen(command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=is_shell_command)
stdout, stderr = process.communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
trace_prefix = "Received from system (%s)" % command
if stdout and stderr:
log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" %
(stdout, stderr))
elif stdout:
log.trace(trace_prefix + ", stdout:\n%s" % stdout)
elif stderr:
log.trace(trace_prefix + ", stderr:\n%s" % stderr)
exit_code = process.poll()
if not ignore_exit_status and exit_code != 0:
raise OSError("%s returned exit status %i" % (command, exit_code))
if stdout:
return stdout.decode("utf-8", "replace").splitlines()
else:
return []
except OSError as exc:
log.debug("System call (failed): %s (error: %s)" % (command, exc))
if default != UNDEFINED:
return default
else:
raise exc
def call(command, default = UNDEFINED):
"""
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
:param str command: command to be issued
:param object default: response if the query fails
:returns: **list** with the lines of output from the command
:raises: **OSError** if this fails and no default was provided
"""
try:
is_shell_command = command.split(" ")[0] in SHELL_COMMANDS
start_time = time.time()
stdout, stderr = subprocess.Popen(command.split(), stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command).communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
trace_prefix = "Received from system (%s)" % command
if stdout and stderr:
log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" % (stdout, stderr))
elif stdout:
log.trace(trace_prefix + ", stdout:\n%s" % stdout)
elif stderr:
log.trace(trace_prefix + ", stderr:\n%s" % stderr)
if stdout:
return stdout.decode("utf-8", "replace").splitlines()
else:
return []
except OSError, exc:
log.debug("System call (failed): %s (error: %s)" % (command, exc))
if default != UNDEFINED:
return default
else:
raise exc
def call(command, suppress_exc = True):
"""
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
:param str command: command to be issued
:param bool suppress_exc: if **True** then **None** is returned on failure,
otherwise this raises the exception
:returns: **list** with the lines of output from the command, **None** in
case of failure if suppress_exc is **True**
:raises: **OSError** if this fails and suppress_exc is **False**
"""
try:
start_time = time.time()
stdout, stderr = subprocess.Popen(command.split(), stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
trace_prefix = "Received from system (%s)" % command
if stdout and stderr:
log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" % (stdout, stderr))
elif stdout:
log.trace(trace_prefix + ", stdout:\n%s" % stdout)
elif stderr:
log.trace(trace_prefix + ", stderr:\n%s" % stderr)
if stdout: return stdout.splitlines()
else: return []
except OSError, exc:
log.debug("System call (failed): %s (error: %s)" % (command, exc))
if suppress_exc: return None
else: raise exc
def recv_message(control_file):
"""
Pulls from a control socket until we either have a complete message or
encounter a problem.
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:returns: :class:`~stem.response.ControlMessage` read from the socket
:raises:
* :class:`stem.ProtocolError` the content from the socket is malformed
* :class:`stem.SocketClosed` if the socket closes before we receive
a complete message
"""
parsed_content, raw_content = [], ""
logging_prefix = "Error while receiving a control message (%s): "
while True:
try:
line = control_file.readline()
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "socket file has been closed")
raise stem.SocketClosed("socket file has been closed")
except socket.error, exc:
# when disconnected we get...
# socket.error: [Errno 107] Transport endpoint is not connected
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "received exception \"%s\"" % exc)
raise stem.SocketClosed(exc)
raw_content += line
# Parses the tor control lines. These are of the form...
# <status code><divider><content>\r\n
if len(line) == 0:
# if the socket is disconnected then the readline() method will provide
# empty content
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "empty socket content")
raise stem.SocketClosed("Received empty socket content.")
elif len(line) < 4:
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "line too short, \"%s\"" % log.escape(line))
raise stem.ProtocolError("Badly formatted reply line: too short")
elif not re.match(r'^[a-zA-Z0-9]{3}[-+ ]', line):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "malformed status code/divider, \"%s\"" %
log.escape(line))
raise stem.ProtocolError(
"Badly formatted reply line: beginning is malformed")
elif not line.endswith("\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
raise stem.ProtocolError("All lines should end with CRLF")
line = line[:-2] # strips off the CRLF
status_code, divider, content = line[:3], line[3], line[4:]
if divider == "-":
# mid-reply line, keep pulling for more content
parsed_content.append((status_code, divider, content))
elif divider == " ":
# end of the message, return the message
parsed_content.append((status_code, divider, content))
log_message = raw_content.replace("\r\n", "\n").rstrip()
log.trace("Received from tor:\n" + log_message)
return stem.response.ControlMessage(parsed_content, raw_content)
elif divider == "+":
# data entry, all of the following lines belong to the content until we
# get a line with just a period
while True:
try:
line = control_file.readline()
except socket.error, exc:
prefix = logging_prefix % "SocketClosed"
log.info(
prefix +
"received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")"
% (exc, log.escape(raw_content)))
raise stem.SocketClosed(exc)
raw_content += line
if not line.endswith("\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(
prefix +
"CRLF linebreaks missing from a data reply, \"%s\"" %
log.escape(raw_content))
raise stem.ProtocolError("All lines should end with CRLF")
elif line == ".\r\n":
break # data block termination
line = line[:-2] # strips off the CRLF
# lines starting with a period are escaped by a second period (as per
# section 2.4 of the control-spec)
if line.startswith(".."):
line = line[1:]
# appends to previous content, using a newline rather than CRLF
# separator (more conventional for multi-line string content outside
# the windows world)
content += "\n" + line
parsed_content.append((status_code, divider, content))
def recv_message(control_file):
"""
Pulls from a control socket until we either have a complete message or
encounter a problem.
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:returns: :class:`~stem.response.ControlMessage` read from the socket
:raises:
* :class:`stem.ProtocolError` the content from the socket is malformed
* :class:`stem.SocketClosed` if the socket closes before we receive
a complete message
"""
parsed_content, raw_content = [], b""
logging_prefix = "Error while receiving a control message (%s): "
while True:
try:
# From a real socket readline() would always provide bytes, but during
# tests we might be given a StringIO in which case it's unicode under
# python 3.x.
line = stem.util.str_tools._to_bytes(control_file.readline())
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "socket file has been closed")
raise stem.SocketClosed("socket file has been closed")
except (socket.error, ValueError) as exc:
# When disconnected we get...
#
# Python 2:
# socket.error: [Errno 107] Transport endpoint is not connected
#
# Python 3:
# ValueError: I/O operation on closed file.
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "received exception \"%s\"" % exc)
raise stem.SocketClosed(exc)
raw_content += line
# Parses the tor control lines. These are of the form...
# <status code><divider><content>\r\n
if len(line) == 0:
# if the socket is disconnected then the readline() method will provide
# empty content
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "empty socket content")
raise stem.SocketClosed("Received empty socket content.")
elif len(line) < 4:
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "line too short, \"%s\"" % log.escape(line))
raise stem.ProtocolError("Badly formatted reply line: too short")
elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "malformed status code/divider, \"%s\"" %
log.escape(line))
raise stem.ProtocolError(
"Badly formatted reply line: beginning is malformed")
elif not line.endswith(b"\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
raise stem.ProtocolError("All lines should end with CRLF")
line = line[:-2] # strips off the CRLF
status_code, divider, content = line[:3], line[3:4], line[4:]
if stem.prereq.is_python_3():
status_code = stem.util.str_tools._to_unicode(status_code)
divider = stem.util.str_tools._to_unicode(divider)
if divider == "-":
# mid-reply line, keep pulling for more content
parsed_content.append((status_code, divider, content))
elif divider == " ":
# end of the message, return the message
parsed_content.append((status_code, divider, content))
log_message = raw_content.replace(b"\r\n", b"\n").rstrip()
log.trace("Received from tor:\n" +
stem.util.str_tools._to_unicode(log_message))
return stem.response.ControlMessage(parsed_content, raw_content)
elif divider == "+":
# data entry, all of the following lines belong to the content until we
# get a line with just a period
while True:
try:
line = stem.util.str_tools._to_bytes(
control_file.readline())
except socket.error as exc:
prefix = logging_prefix % "SocketClosed"
log.info(
prefix +
"received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")"
% (exc, log.escape(raw_content)))
raise stem.SocketClosed(exc)
raw_content += line
if not line.endswith(b"\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(
prefix +
"CRLF linebreaks missing from a data reply, \"%s\"" %
log.escape(raw_content))
raise stem.ProtocolError("All lines should end with CRLF")
elif line == b".\r\n":
break # data block termination
line = line[:-2] # strips off the CRLF
# lines starting with a period are escaped by a second period (as per
# section 2.4 of the control-spec)
if line.startswith(b".."):
line = line[1:]
# appends to previous content, using a newline rather than CRLF
# separator (more conventional for multi-line string content outside
# the windows world)
content += b"\n" + line
parsed_content.append((status_code, divider, content))
else:
# this should never be reached due to the prefix regex, but might as well
# be safe...
prefix = logging_prefix % "ProtocolError"
log.warn(prefix +
"\"%s\" isn't a recognized divider type" % divider)
raise stem.ProtocolError(
"Unrecognized divider type '%s': %s" %
(divider, stem.util.str_tools._to_unicode(line)))
def call(command, default=UNDEFINED, ignore_exit_status=False, env=None):
"""
call(command, default = UNDEFINED, ignore_exit_status = False)
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
.. versionchanged:: 1.5.0
Providing additional information upon failure by raising a CallError. This
is a subclass of OSError, providing backward compatibility.
.. versionchanged:: 1.5.0
Added env argument.
:param str,list command: command to be issued
:param object default: response if the query fails
:param bool ignore_exit_status: reports failure if our command's exit status
was non-zero
:param dict env: environment variables
:returns: **list** with the lines of output from the command
:raises: **CallError** if this fails and no default was provided, this is an
**OSError** subclass
"""
global SYSTEM_CALL_TIME
if isinstance(command, str):
command_list = command.split(' ')
else:
command_list = command
exit_status, runtime, stdout, stderr = None, None, None, None
start_time = time.time()
try:
is_shell_command = command_list[0] in SHELL_COMMANDS
process = subprocess.Popen(command_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=is_shell_command,
env=env)
stdout, stderr = process.communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
trace_prefix = 'Received from system (%s)' % command
if stdout and stderr:
log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' %
(stdout, stderr))
elif stdout:
log.trace(trace_prefix + ', stdout:\n%s' % stdout)
elif stderr:
log.trace(trace_prefix + ', stderr:\n%s' % stderr)
exit_status = process.poll()
if not ignore_exit_status and exit_status != 0:
raise OSError('%s returned exit status %i' %
(command, exit_status))
if stdout:
return stdout.decode('utf-8', 'replace').splitlines()
else:
return []
except OSError as exc:
log.debug('System call (failed): %s (error: %s)' % (command, exc))
if default != UNDEFINED:
return default
else:
raise CallError(str(exc), ' '.join(command_list), exit_status,
runtime, stdout, stderr)
finally:
with SYSTEM_CALL_TIME_LOCK:
SYSTEM_CALL_TIME += time.time() - start_time
def get_info(self,
params,
default=stem.control.UNDEFINED,
get_bytes=False,
cache_miss_warning=True):
"""
get_info(params, default = UNDEFINED, get_bytes = False)
Queries the control socket for the given GETINFO option. If provided a
default then that's returned if the GETINFO option is undefined or the
call fails for any reason (error response, control port closed, initiated,
etc).
.. versionchanged:: 1.1.0
Added the get_bytes argument.
:param str,list params: GETINFO option or options to be queried
:param object default: response if the query fails
:param bool get_bytes: provides **bytes** values rather than a **str** under python 3.x
:param bool cache_miss_warning: Emit a warning if a cache miss happens
:returns:
Response depends upon how we were called as follows...
* **str** with the response if our param was a **str**
* **dict** with the 'param => response' mapping if our param was a **list**
* default if one was provided and our call failed
:raises:
* :class:`stem.ControllerError` if the call fails and we weren't
provided a default response
* :class:`stem.InvalidArguments` if the 'params' requested was
invalid
* :class:`stem.ProtocolError` if the geoip database is known to be
unavailable
"""
start_time = time.time()
reply = {}
if isinstance(params, (bytes, str_type)):
is_multiple = False
params = set([params])
else:
if not params:
return {}
is_multiple = True
params = set(params)
# check for cached results
from_cache = [param.lower() for param in params]
cached_results = self._get_cache_map(from_cache, 'getinfo')
for key in cached_results:
user_expected_key = stem.control._case_insensitive_lookup(
params, key)
reply[user_expected_key] = cached_results[key]
params.remove(user_expected_key)
for param in params:
if param.startswith(
'ip-to-country/') and self.is_geoip_unavailable():
# the geoip database already looks to be unavailable - abort the request
raise stem.ProtocolError('Tor geoip database is unavailable')
# if everything was cached then short circuit making the query
if not params:
if stem.control.LOG_CACHE_FETCHES:
log.trace('GETINFO %s (cache fetch)' % ' '.join(reply.keys()))
if is_multiple:
return reply
else:
return list(reply.values())[0]
if cache_miss_warning is True:
# As this should only happen when we intentionally refresh the cache (when the warning should be disabled)
# we warn rather than debug to ensure we get the issue!
lgr = logging.getLogger('theonionbox')
lgr.info(
'Cache miss for the following parameter(s): {}'.format(params))
try:
response = self.msg('GETINFO %s' % ' '.join(params))
stem.response.convert('GETINFO', response)
response._assert_matches(params)
# usually we want unicode values under python 3.x
if stem.prereq.is_python_3() and not get_bytes:
response.entries = dict((k, stem.util.str_tools._to_unicode(v))
for (k, v) in response.entries.items())
reply.update(response.entries)
if self.is_caching_enabled():
to_cache = {}
for key, value in response.entries.items():
key = key.lower() # make case insensitive
# To allow The Onion Box smooth response cycles even when connecting to a relay / bridge
# via Tor socks proxy / hidden service, we cache *all* parameters and 'manually' update them with a
# single call every once in a while.
to_cache[key] = value
if key.startswith('ip-to-country/'):
# both cache-able and means that we should reset the geoip failure count
self._geoip_failure_count = -1
self._set_cache(to_cache, 'getinfo')
log.debug('GETINFO %s (runtime: %0.4f)' %
(' '.join(params), time.time() - start_time))
if is_multiple:
return reply
else:
return list(reply.values())[0]
except stem.ControllerError as exc:
# bump geoip failure count if...
# * we're caching results
# * this was soley a geoip lookup
# * we've never had a successful geoip lookup (failure count isn't -1)
is_geoip_request = len(params) == 1 and list(params)[0].startswith(
'ip-to-country/')
if is_geoip_request and self.is_caching_enabled(
) and self._geoip_failure_count != -1:
self._geoip_failure_count += 1
if self.is_geoip_unavailable():
log.warn("Tor's geoip database is unavailable.")
log.debug('GETINFO %s (failed: %s)' % (' '.join(params), exc))
raise
def run(self):
while not self._halt:
minWait = self.resolveRate if self.resolveRate else self.defaultRate
timeSinceReset = time.time() - self.lastLookup
if self._isPaused or timeSinceReset < minWait:
sleepTime = max(0.2, minWait - timeSinceReset)
self._cond.acquire()
if not self._halt: self._cond.wait(sleepTime)
self._cond.release()
continue # done waiting, try again
isDefault = self.overwriteResolver == None
resolver = self.defaultResolver if isDefault else self.overwriteResolver
# checks if there's nothing to resolve with
if not resolver:
self.lastLookup = time.time() # avoids a busy wait in this case
continue
try:
resolveStart = time.time()
connResults = getConnections(resolver, self.processName, self.processPid)
lookupTime = time.time() - resolveStart
self._connections = connResults
self._resolutionCounter += 1
newMinDefaultRate = 100 * lookupTime
if self.defaultRate < newMinDefaultRate:
if self._rateThresholdBroken >= 3:
# adding extra to keep the rate from frequently changing
self.defaultRate = newMinDefaultRate + 0.5
log.trace("connection lookup time increasing to %0.1f seconds per call" % self.defaultRate)
else: self._rateThresholdBroken += 1
else: self._rateThresholdBroken = 0
if isDefault: self._subsiquentFailures = 0
except (ValueError, IOError), exc:
# this logs in a couple of cases:
# - special failures noted by getConnections (most cases are already
# logged via system)
# - note fail-overs for default resolution methods
if str(exc).startswith("No results found using:"):
log.info(exc)
if isDefault:
self._subsiquentFailures += 1
if self._subsiquentFailures >= RESOLVER_FAILURE_TOLERANCE:
# failed several times in a row - abandon resolver and move on to another
self._resolverBlacklist.append(resolver)
self._subsiquentFailures = 0
# pick another (non-blacklisted) resolver
newResolver = None
for r in self.resolverOptions:
if not r in self._resolverBlacklist:
newResolver = r
break
if newResolver:
# provide notice that failures have occurred and resolver is changing
log.notice(RESOLVER_SERIAL_FAILURE_MSG % (resolver, newResolver))
else:
# exhausted all resolvers, give warning
log.notice(RESOLVER_FINAL_FAILURE_MSG)
self.defaultResolver = newResolver
finally:
def recv_message(control_file):
"""
Pulls from a control socket until we either have a complete message or
encounter a problem.
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:returns: :class:`~stem.response.ControlMessage` read from the socket
:raises:
* :class:`stem.ProtocolError` the content from the socket is malformed
* :class:`stem.SocketClosed` if the socket closes before we receive
a complete message
"""
parsed_content, raw_content = [], []
logging_prefix = 'Error while receiving a control message (%s): '
while True:
try:
# From a real socket readline() would always provide bytes, but during
# tests we might be given a StringIO in which case it's unicode under
# python 3.x.
line = stem.util.str_tools._to_bytes(control_file.readline())
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'socket file has been closed')
raise stem.SocketClosed('socket file has been closed')
except (socket.error, ValueError) as exc:
# When disconnected we get...
#
# Python 2:
# socket.error: [Errno 107] Transport endpoint is not connected
#
# Python 3:
# ValueError: I/O operation on closed file.
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'received exception "%s"' % exc)
raise stem.SocketClosed(exc)
raw_content.append(line)
# Parses the tor control lines. These are of the form...
# <status code><divider><content>\r\n
if len(line) == 0:
# if the socket is disconnected then the readline() method will provide
# empty content
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'empty socket content')
raise stem.SocketClosed('Received empty socket content.')
elif len(line) < 4:
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'line too short, "%s"' % log.escape(line))
raise stem.ProtocolError('Badly formatted reply line: too short')
elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
prefix = logging_prefix % 'ProtocolError'
log.info(prefix +
'malformed status code/divider, "%s"' % log.escape(line))
raise stem.ProtocolError(
'Badly formatted reply line: beginning is malformed')
elif not line.endswith(b'\r\n'):
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'no CRLF linebreak, "%s"' % log.escape(line))
raise stem.ProtocolError('All lines should end with CRLF')
line = line[:-2] # strips off the CRLF
status_code, divider, content = line[:3], line[3:4], line[4:]
content_lines = [content]
if stem.prereq.is_python_3():
status_code = stem.util.str_tools._to_unicode(status_code)
divider = stem.util.str_tools._to_unicode(divider)
if divider == '-':
# mid-reply line, keep pulling for more content
parsed_content.append((status_code, divider, content))
elif divider == ' ':
# end of the message, return the message
parsed_content.append((status_code, divider, content))
raw_content_str = b''.join(raw_content)
log_message = stem.util.str_tools._to_unicode(
raw_content_str.replace(b'\r\n', b'\n').rstrip())
log_message_lines = log_message.split('\n')
if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS:
log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + [
'... %i more lines...' %
(len(log_message_lines) - TRUNCATE_LOGS)
])
if len(log_message_lines) > 2:
log.trace('Received from tor:\n%s' % log_message)
else:
log.trace('Received from tor: %s' %
log_message.replace('\n', '\\n'))
return stem.response.ControlMessage(parsed_content,
raw_content_str)
elif divider == '+':
# data entry, all of the following lines belong to the content until we
# get a line with just a period
while True:
try:
line = stem.util.str_tools._to_bytes(
control_file.readline())
except socket.error as exc:
prefix = logging_prefix % 'SocketClosed'
log.info(
prefix +
'received an exception while mid-way through a data reply (exception: "%s", read content: "%s")'
% (exc, log.escape(b''.join(raw_content))))
raise stem.SocketClosed(exc)
raw_content.append(line)
if not line.endswith(b'\r\n'):
prefix = logging_prefix % 'ProtocolError'
log.info(
prefix +
'CRLF linebreaks missing from a data reply, "%s"' %
log.escape(b''.join(raw_content)))
raise stem.ProtocolError('All lines should end with CRLF')
elif line == b'.\r\n':
break # data block termination
line = line[:-2] # strips off the CRLF
# lines starting with a period are escaped by a second period (as per
# section 2.4 of the control-spec)
if line.startswith(b'..'):
line = line[1:]
content_lines.append(line)
# joins the content using a newline rather than CRLF separator (more
# conventional for multi-line string content outside the windows world)
parsed_content.append(
(status_code, divider, b'\n'.join(content_lines)))
else:
# this should never be reached due to the prefix regex, but might as well
# be safe...
prefix = logging_prefix % 'ProtocolError'
log.warn(prefix +
"\"%s\" isn't a recognized divider type" % divider)
raise stem.ProtocolError(
"Unrecognized divider type '%s': %s" %
(divider, stem.util.str_tools._to_unicode(line)))
def recv_message(control_file):
"""
Pulls from a control socket until we either have a complete message or
encounter a problem.
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:returns: :class:`~stem.response.ControlMessage` read from the socket
:raises:
* :class:`stem.ProtocolError` the content from the socket is malformed
* :class:`stem.SocketClosed` if the socket closes before we receive
a complete message
"""
parsed_content, raw_content = [], ""
logging_prefix = "Error while receiving a control message (%s): "
while True:
try:
line = control_file.readline()
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "socket file has been closed")
raise stem.SocketClosed("socket file has been closed")
except socket.error, exc:
# when disconnected we get...
# socket.error: [Errno 107] Transport endpoint is not connected
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "received exception \"%s\"" % exc)
raise stem.SocketClosed(exc)
raw_content += line
# Parses the tor control lines. These are of the form...
# <status code><divider><content>\r\n
if len(line) == 0:
# if the socket is disconnected then the readline() method will provide
# empty content
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "empty socket content")
raise stem.SocketClosed("Received empty socket content.")
elif len(line) < 4:
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "line too short, \"%s\"" % log.escape(line))
raise stem.ProtocolError("Badly formatted reply line: too short")
elif not re.match(r'^[a-zA-Z0-9]{3}[-+ ]', line):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "malformed status code/divider, \"%s\"" % log.escape(line))
raise stem.ProtocolError("Badly formatted reply line: beginning is malformed")
elif not line.endswith("\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
raise stem.ProtocolError("All lines should end with CRLF")
line = line[:-2] # strips off the CRLF
status_code, divider, content = line[:3], line[3], line[4:]
if divider == "-":
# mid-reply line, keep pulling for more content
parsed_content.append((status_code, divider, content))
elif divider == " ":
# end of the message, return the message
parsed_content.append((status_code, divider, content))
log_message = raw_content.replace("\r\n", "\n").rstrip()
log.trace("Received from tor:\n" + log_message)
return stem.response.ControlMessage(parsed_content, raw_content)
elif divider == "+":
# data entry, all of the following lines belong to the content until we
# get a line with just a period
while True:
try:
line = control_file.readline()
except socket.error, exc:
prefix = logging_prefix % "SocketClosed"
log.info(prefix + "received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")" % (exc, log.escape(raw_content)))
raise stem.SocketClosed(exc)
raw_content += line
if not line.endswith("\r\n"):
prefix = logging_prefix % "ProtocolError"
log.info(prefix + "CRLF linebreaks missing from a data reply, \"%s\"" % log.escape(raw_content))
raise stem.ProtocolError("All lines should end with CRLF")
elif line == ".\r\n":
break # data block termination
line = line[:-2] # strips off the CRLF
# lines starting with a period are escaped by a second period (as per
# section 2.4 of the control-spec)
if line.startswith(".."):
line = line[1:]
# appends to previous content, using a newline rather than CRLF
# separator (more conventional for multi-line string content outside
# the windows world)
content += "\n" + line
parsed_content.append((status_code, divider, content))
def recv_message(control_file):
"""
Pulls from a control socket until we either have a complete message or
encounter a problem.
:param file control_file: file derived from the control socket (see the
socket's makefile() method for more information)
:returns: :class:`~stem.response.ControlMessage` read from the socket
:raises:
* :class:`stem.ProtocolError` the content from the socket is malformed
* :class:`stem.SocketClosed` if the socket closes before we receive
a complete message
"""
parsed_content, raw_content = [], []
logging_prefix = 'Error while receiving a control message (%s): '
while True:
try:
# From a real socket readline() would always provide bytes, but during
# tests we might be given a StringIO in which case it's unicode under
# python 3.x.
line = stem.util.str_tools._to_bytes(control_file.readline())
except AttributeError:
# if the control_file has been closed then we will receive:
# AttributeError: 'NoneType' object has no attribute 'recv'
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'socket file has been closed')
raise stem.SocketClosed('socket file has been closed')
except (socket.error, ValueError) as exc:
# When disconnected we get...
#
# Python 2:
# socket.error: [Errno 107] Transport endpoint is not connected
#
# Python 3:
# ValueError: I/O operation on closed file.
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'received exception "%s"' % exc)
raise stem.SocketClosed(exc)
raw_content.append(line)
# Parses the tor control lines. These are of the form...
# <status code><divider><content>\r\n
if len(line) == 0:
# if the socket is disconnected then the readline() method will provide
# empty content
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'empty socket content')
raise stem.SocketClosed('Received empty socket content.')
elif len(line) < 4:
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'line too short, "%s"' % log.escape(line))
raise stem.ProtocolError('Badly formatted reply line: too short')
elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'malformed status code/divider, "%s"' % log.escape(line))
raise stem.ProtocolError('Badly formatted reply line: beginning is malformed')
elif not line.endswith(b'\r\n'):
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'no CRLF linebreak, "%s"' % log.escape(line))
raise stem.ProtocolError('All lines should end with CRLF')
line = line[:-2] # strips off the CRLF
status_code, divider, content = line[:3], line[3:4], line[4:]
content_lines = [content]
if stem.prereq.is_python_3():
status_code = stem.util.str_tools._to_unicode(status_code)
divider = stem.util.str_tools._to_unicode(divider)
if divider == '-':
# mid-reply line, keep pulling for more content
parsed_content.append((status_code, divider, content))
elif divider == ' ':
# end of the message, return the message
parsed_content.append((status_code, divider, content))
raw_content_str = b''.join(raw_content)
log_message = raw_content_str.replace(b'\r\n', b'\n').rstrip()
log.trace('Received from tor:\n' + stem.util.str_tools._to_unicode(log_message))
return stem.response.ControlMessage(parsed_content, raw_content_str)
elif divider == '+':
# data entry, all of the following lines belong to the content until we
# get a line with just a period
while True:
try:
line = stem.util.str_tools._to_bytes(control_file.readline())
except socket.error as exc:
prefix = logging_prefix % 'SocketClosed'
log.info(prefix + 'received an exception while mid-way through a data reply (exception: "%s", read content: "%s")' % (exc, log.escape(b''.join(raw_content))))
raise stem.SocketClosed(exc)
raw_content.append(line)
if not line.endswith(b'\r\n'):
prefix = logging_prefix % 'ProtocolError'
log.info(prefix + 'CRLF linebreaks missing from a data reply, "%s"' % log.escape(b''.join(raw_content)))
raise stem.ProtocolError('All lines should end with CRLF')
elif line == b'.\r\n':
break # data block termination
line = line[:-2] # strips off the CRLF
# lines starting with a period are escaped by a second period (as per
# section 2.4 of the control-spec)
if line.startswith(b'..'):
line = line[1:]
content_lines.append(line)
# joins the content using a newline rather than CRLF separator (more
# conventional for multi-line string content outside the windows world)
parsed_content.append((status_code, divider, b'\n'.join(content_lines)))
else:
# this should never be reached due to the prefix regex, but might as well
# be safe...
prefix = logging_prefix % 'ProtocolError'
log.warn(prefix + "\"%s\" isn't a recognized divider type" % divider)
raise stem.ProtocolError("Unrecognized divider type '%s': %s" % (divider, stem.util.str_tools._to_unicode(line)))
def call(command, default = UNDEFINED, ignore_exit_status = False, timeout = None, cwd = None, env = None):
"""
call(command, default = UNDEFINED, ignore_exit_status = False)
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
.. versionchanged:: 1.5.0
Providing additional information upon failure by raising a CallError. This
is a subclass of OSError, providing backward compatibility.
.. versionchanged:: 1.5.0
Added env argument.
.. versionchanged:: 1.6.0
Added timeout and cwd arguments.
:param str,list command: command to be issued
:param object default: response if the query fails
:param bool ignore_exit_status: reports failure if our command's exit status
was non-zero
:param float timeout: maximum seconds to wait, blocks indefinitely if
**None**
:param dict env: environment variables
:returns: **list** with the lines of output from the command
:raises:
* **CallError** if this fails and no default was provided
* **CallTimeoutError** if the timeout is reached without a default
"""
global SYSTEM_CALL_TIME
if isinstance(command, str):
command_list = command.split(' ')
else:
command_list = list(map(str, command))
exit_status, runtime, stdout, stderr = None, None, None, None
start_time = time.time()
try:
is_shell_command = command_list[0] in SHELL_COMMANDS
process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, cwd = cwd, env = env)
if timeout:
while process.poll() is None:
if time.time() - start_time > timeout:
raise CallTimeoutError("Process didn't finish after %0.1f seconds" % timeout, ' '.join(command_list), None, timeout, '', '', timeout)
time.sleep(0.001)
stdout, stderr = process.communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
trace_prefix = 'Received from system (%s)' % command
if stdout and stderr:
log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
elif stdout:
log.trace(trace_prefix + ', stdout:\n%s' % stdout)
elif stderr:
log.trace(trace_prefix + ', stderr:\n%s' % stderr)
exit_status = process.poll()
if not ignore_exit_status and exit_status != 0:
raise OSError('%s returned exit status %i' % (command, exit_status))
if stdout:
return stdout.decode('utf-8', 'replace').splitlines()
else:
return []
except CallTimeoutError as exc:
log.debug('System call (timeout): %s (after %0.4fs)' % (command, timeout))
if default != UNDEFINED:
return default
else:
raise
except OSError as exc:
log.debug('System call (failed): %s (error: %s)' % (command, exc))
if default != UNDEFINED:
return default
else:
raise CallError(str(exc), ' '.join(command_list), exit_status, runtime, stdout, stderr)
finally:
with SYSTEM_CALL_TIME_LOCK:
SYSTEM_CALL_TIME += time.time() - start_time
def call(command, default = UNDEFINED, ignore_exit_status = False, timeout = None, cwd = None, env = None):
"""
call(command, default = UNDEFINED, ignore_exit_status = False)
Issues a command in a subprocess, blocking until completion and returning the
results. This is not actually ran in a shell so pipes and other shell syntax
are not permitted.
.. versionchanged:: 1.5.0
Providing additional information upon failure by raising a CallError. This
is a subclass of OSError, providing backward compatibility.
.. versionchanged:: 1.5.0
Added env argument.
.. versionchanged:: 1.6.0
Added timeout and cwd arguments.
:param str,list command: command to be issued
:param object default: response if the query fails
:param bool ignore_exit_status: reports failure if our command's exit status
was non-zero
:param float timeout: maximum seconds to wait, blocks indefinitely if
**None**
:param dict env: environment variables
:returns: **list** with the lines of output from the command
:raises:
* **CallError** if this fails and no default was provided
* **CallTimeoutError** if the timeout is reached without a default
"""
# TODO: in stem 2.x return a struct with stdout, stderr, and runtime instead
global SYSTEM_CALL_TIME
if isinstance(command, str):
command_list = command.split(' ')
else:
command_list = list(map(str, command))
exit_status, runtime, stdout, stderr = None, None, None, None
start_time = time.time()
try:
is_shell_command = command_list[0] in SHELL_COMMANDS
process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, cwd = cwd, env = env)
if timeout:
while process.poll() is None:
if time.time() - start_time > timeout:
raise CallTimeoutError("Process didn't finish after %0.1f seconds" % timeout, ' '.join(command_list), None, timeout, '', '', timeout)
time.sleep(0.001)
stdout, stderr = process.communicate()
stdout, stderr = stdout.strip(), stderr.strip()
runtime = time.time() - start_time
log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
if log.is_tracing():
trace_prefix = 'Received from system (%s)' % command
if stdout and stderr:
log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
elif stdout:
log.trace(trace_prefix + ', stdout:\n%s' % stdout)
elif stderr:
log.trace(trace_prefix + ', stderr:\n%s' % stderr)
exit_status = process.poll()
if not ignore_exit_status and exit_status != 0:
raise OSError('%s returned exit status %i' % (command, exit_status))
if stdout:
return stdout.decode('utf-8', 'replace').splitlines()
else:
return []
except CallTimeoutError:
log.debug('System call (timeout): %s (after %0.4fs)' % (command, timeout))
if default != UNDEFINED:
return default
else:
raise
except OSError as exc:
log.debug('System call (failed): %s (error: %s)' % (command, exc))
if default != UNDEFINED:
return default
else:
raise CallError(str(exc), ' '.join(command_list), exit_status, runtime, stdout, stderr)
finally:
with SYSTEM_CALL_TIME_LOCK:
SYSTEM_CALL_TIME += time.time() - start_time