Пример #1
0
  def _download_descriptors(self, retries):
    try:
      use_authority = retries == 0 and self.fall_back_to_authority
      self.download_url = self._pick_url(use_authority)

      self.start_time = time.time()
      response = urllib2.urlopen(self.download_url, timeout = self.timeout).read()

      if self.download_url.endswith('.z'):
        response = zlib.decompress(response)

      self.content = response.strip()

      self.runtime = time.time() - self.start_time
      log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
    except:
      exc = sys.exc_info()[1]

      if retries > 0:
        log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
        return self._download_descriptors(retries - 1)
      else:
        log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
        self.error = exc
    finally:
      self.is_done = True
Пример #2
0
def send_message(control_file, message, raw=False):
    """
  Sends a message to the control socket, adding the expected formatting for
  single verses multi-line messages. Neither message type should contain an
  ending newline (if so it'll be treated as a multi-line message with a blank
  line at the end). If the message doesn't contain a newline then it's sent
  as...

  ::

    <message>\\r\\n

  and if it does contain newlines then it's split on ``\\n`` and sent as...

  ::

    +<line 1>\\r\\n
    <line 2>\\r\\n
    <line 3>\\r\\n
    .\\r\\n

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)
  :param str message: message to be sent on the control socket
  :param bool raw: leaves the message formatting untouched, passing it to the
    socket as-is

  :raises:
    * :class:`stem.SocketError` if a problem arises in using the socket
    * :class:`stem.SocketClosed` if the socket is known to be shut down
  """

    if not raw:
        message = send_formatting(message)

    try:
        control_file.write(stem.util.str_tools._to_bytes(message))
        control_file.flush()

        if log.is_tracing():
            log_message = message.replace('\r\n', '\n').rstrip()
            msg_div = '\n' if '\n' in log_message else ' '
            log.trace('Sent to tor:%s%s' % (msg_div, log_message))
    except socket.error as exc:
        log.info('Failed to send message: %s' % exc)

        # When sending there doesn't seem to be a reliable method for
        # distinguishing between failures from a disconnect verses other things.
        # Just accounting for known disconnection responses.

        if str(exc) == '[Errno 32] Broken pipe':
            raise stem.SocketClosed(exc)
        else:
            raise stem.SocketError(exc)
    except AttributeError:
        # if the control_file has been closed then flush will receive:
        # AttributeError: 'NoneType' object has no attribute 'sendall'

        log.info('Failed to send message: file has been closed')
        raise stem.SocketClosed('file has been closed')
Пример #3
0
  def _download_descriptors(self, retries):
    try:
      use_authority = retries == 0 and self.fall_back_to_authority
      self.download_url = self._pick_url(use_authority)

      self.start_time = time.time()
      response = urllib2.urlopen(self.download_url, timeout = self.timeout).read()

      if self.download_url.endswith('.z'):
        response = zlib.decompress(response)

      self.content = response.strip()

      self.runtime = time.time() - self.start_time
      log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
    except:
      exc = sys.exc_info()[1]

      if retries > 0:
        log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
        return self._download_descriptors(retries - 1)
      else:
        log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
        self.error = exc
    finally:
      self.is_done = True
Пример #4
0
  def _download_descriptors(self, retries, timeout):
    try:
      self.start_time = time.time()
      endpoint = self._pick_endpoint(use_authority = retries == 0 and self.fall_back_to_authority)

      if isinstance(endpoint, stem.ORPort):
        self.content, self.reply_headers = _download_from_orport(endpoint, self.compression, self.resource)
      elif isinstance(endpoint, stem.DirPort):
        self.download_url = 'http://%s:%i/%s' % (endpoint.address, endpoint.port, self.resource.lstrip('/'))
        self.content, self.reply_headers = _download_from_dirport(self.download_url, self.compression, timeout)
      else:
        raise ValueError("BUG: endpoints can only be ORPorts or DirPorts, '%s' was a %s" % (endpoint, type(endpoint).__name__))

      self.runtime = time.time() - self.start_time
      log.trace("Descriptors retrieved from '%s' in %0.2fs" % (self.download_url, self.runtime))
    except:
      exc = sys.exc_info()[1]

      if timeout is not None:
        timeout -= time.time() - self.start_time

      if retries > 0 and (timeout is None or timeout > 0):
        log.debug("Unable to download descriptors from '%s' (%i retries remaining): %s" % (self.download_url, retries, exc))
        return self._download_descriptors(retries - 1, timeout)
      else:
        log.debug("Unable to download descriptors from '%s': %s" % (self.download_url, exc))
        self.error = exc
    finally:
      self.is_done = True
Пример #5
0
    async def _download_descriptors(self, retries: int,
                                    timeout: Optional[float]) -> None:
        self.start_time = time.time()

        retries = self.retries
        time_remaining = self.timeout

        while True:
            endpoint = self._pick_endpoint(
                use_authority=retries == 0 and self.fall_back_to_authority)

            if isinstance(endpoint, stem.ORPort):
                downloaded_from = 'ORPort %s:%s (resource %s)' % (
                    endpoint.address, endpoint.port, self.resource)
            elif isinstance(endpoint, stem.DirPort):
                downloaded_from = 'http://%s:%i/%s' % (
                    endpoint.address, endpoint.port, self.resource.lstrip('/'))
                self.download_url = downloaded_from
            else:
                raise ValueError(
                    "BUG: endpoints can only be ORPorts or DirPorts, '%s' was a %s"
                    % (endpoint, type(endpoint).__name__))

            try:
                response = await asyncio.wait_for(
                    self._download_from(endpoint), time_remaining)
                self.content, self.reply_headers = _http_body_and_headers(
                    response)

                self.is_done = True
                self.runtime = time.time() - self.start_time

                log.trace('Descriptors retrieved from %s in %0.2fs' %
                          (downloaded_from, self.runtime))
                return
            except asyncio.TimeoutError as exc:
                self.is_done = True
                self.error = stem.DownloadTimeout(downloaded_from, exc,
                                                  sys.exc_info()[2],
                                                  self.timeout)
                return
            except:
                exception = sys.exc_info()[1]
                retries -= 1

                if time_remaining is not None:
                    time_remaining -= time.time() - self.start_time

                if retries > 0:
                    log.debug(
                        "Failed to download descriptors from '%s' (%i retries remaining): %s"
                        % (downloaded_from, retries, exception))
                else:
                    log.debug("Failed to download descriptors from '%s': %s" %
                              (self.download_url, exception))

                    self.is_done = True
                    self.error = exception
                    return
Пример #6
0
def send_message(control_file, message, raw = False):
  """
  Sends a message to the control socket, adding the expected formatting for
  single verses multi-line messages. Neither message type should contain an
  ending newline (if so it'll be treated as a multi-line message with a blank
  line at the end). If the message doesn't contain a newline then it's sent
  as...

  ::

    <message>\\r\\n

  and if it does contain newlines then it's split on ``\\n`` and sent as...

  ::

    +<line 1>\\r\\n
    <line 2>\\r\\n
    <line 3>\\r\\n
    .\\r\\n

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)
  :param str message: message to be sent on the control socket
  :param bool raw: leaves the message formatting untouched, passing it to the
    socket as-is

  :raises:
    * :class:`stem.SocketError` if a problem arises in using the socket
    * :class:`stem.SocketClosed` if the socket is known to be shut down
  """

  if not raw:
    message = send_formatting(message)

  try:
    control_file.write(stem.util.str_tools._to_bytes(message))
    control_file.flush()

    log_message = message.replace('\r\n', '\n').rstrip()
    log.trace('Sent to tor:\n' + log_message)
  except socket.error as exc:
    log.info('Failed to send message: %s' % exc)

    # When sending there doesn't seem to be a reliable method for
    # distinguishing between failures from a disconnect verses other things.
    # Just accounting for known disconnection responses.

    if str(exc) == '[Errno 32] Broken pipe':
      raise stem.SocketClosed(exc)
    else:
      raise stem.SocketError(exc)
  except AttributeError:
    # if the control_file has been closed then flush will receive:
    # AttributeError: 'NoneType' object has no attribute 'sendall'

    log.info('Failed to send message: file has been closed')
    raise stem.SocketClosed('file has been closed')
Пример #7
0
def send_message(control_file, message, raw=False):
    """
  Sends a message to the control socket, adding the expected formatting for
  single verses multi-line messages. Neither message type should contain an
  ending newline (if so it'll be treated as a multi-line message with a blank
  line at the end). If the message doesn't contain a newline then it's sent
  as...

  ::

    <message>\\r\\n

  and if it does contain newlines then it's split on ``\\n`` and sent as...

  ::

    +<line 1>\\r\\n
    <line 2>\\r\\n
    <line 3>\\r\\n
    .\\r\\n

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)
  :param str message: message to be sent on the control socket
  :param bool raw: leaves the message formatting untouched, passing it to the
    socket as-is

  :raises:
    * :class:`stem.SocketError` if a problem arises in using the socket
    * :class:`stem.SocketClosed` if the socket is known to be shut down
  """

    if not raw:
        message = send_formatting(message)

    try:
        control_file.write(message)
        control_file.flush()

        log_message = message.replace("\r\n", "\n").rstrip()
        log.trace("Sent to tor:\n" + log_message)
    except socket.error, exc:
        log.info("Failed to send message: %s" % exc)

        # When sending there doesn't seem to be a reliable method for
        # distinguishing between failures from a disconnect verses other things.
        # Just accounting for known disconnection responses.

        if str(exc) == "[Errno 32] Broken pipe":
            raise stem.SocketClosed(exc)
        else:
            raise stem.SocketError(exc)
Пример #8
0
def _log_trace(response):
  if not log.is_tracing():
    return

  log_message = stem.util.str_tools._to_unicode(response.replace(b'\r\n', b'\n').rstrip())
  log_message_lines = log_message.split('\n')

  if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS:
    log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + ['... %i more lines...' % (len(log_message_lines) - TRUNCATE_LOGS)])

  if len(log_message_lines) > 2:
    log.trace('Received from tor:\n%s' % log_message)
  else:
    log.trace('Received from tor: %s' % log_message.replace('\n', '\\n'))
Пример #9
0
async def send_message(writer: asyncio.StreamWriter,
                       message: Union[bytes, str],
                       raw: bool = False) -> None:
    """
  Sends a message to the control socket, adding the expected formatting for
  single verses multi-line messages. Neither message type should contain an
  ending newline (if so it'll be treated as a multi-line message with a blank
  line at the end). If the message doesn't contain a newline then it's sent
  as...

  ::

    <message>\\r\\n

  and if it does contain newlines then it's split on ``\\n`` and sent as...

  ::

    +<line 1>\\r\\n
    <line 2>\\r\\n
    <line 3>\\r\\n
    .\\r\\n

  :param writer: writer object
  :param message: message to be sent on the control socket
  :param raw: leaves the message formatting untouched, passing it to the
    socket as-is

  :raises:
    * :class:`stem.SocketError` if a problem arises in using the socket
    * :class:`stem.SocketClosed` if the socket is known to be shut down
  """

    message = stem.util.str_tools._to_unicode(message)

    if not raw:
        message = send_formatting(message)

    await _write_to_socket(writer, message)

    if log.is_tracing():
        log_message = message.replace('\r\n', '\n').rstrip()
        msg_div = '\n' if '\n' in log_message else ' '
        log.trace('Sent to tor:%s%s' % (msg_div, log_message))
Пример #10
0
def send_message(control_file, message, raw = False):
  """
  Sends a message to the control socket, adding the expected formatting for
  single verses multi-line messages. Neither message type should contain an
  ending newline (if so it'll be treated as a multi-line message with a blank
  line at the end). If the message doesn't contain a newline then it's sent
  as...

  ::

    <message>\\r\\n

  and if it does contain newlines then it's split on ``\\n`` and sent as...

  ::

    +<line 1>\\r\\n
    <line 2>\\r\\n
    <line 3>\\r\\n
    .\\r\\n

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)
  :param str message: message to be sent on the control socket
  :param bool raw: leaves the message formatting untouched, passing it to the
    socket as-is

  :raises:
    * :class:`stem.SocketError` if a problem arises in using the socket
    * :class:`stem.SocketClosed` if the socket is known to be shut down
  """

  if not raw:
    message = send_formatting(message)

  _write_to_socket(control_file, message)

  if log.is_tracing():
    log_message = message.replace('\r\n', '\n').rstrip()
    msg_div = '\n' if '\n' in log_message else ' '
    log.trace('Sent to tor:%s%s' % (msg_div, log_message))
Пример #11
0
def _setup_debug_logging(args):
  """
  Configures us to log at stem's trace level to a debug log path. This starts
  it off with some general diagnostic information.
  """

  debug_dir = os.path.dirname(args.debug_path)

  if not os.path.exists(debug_dir):
    os.makedirs(debug_dir)

  debug_handler = logging.FileHandler(args.debug_path, mode = 'w')
  debug_handler.setLevel(stem.util.log.logging_level(stem.util.log.TRACE))
  debug_handler.setFormatter(logging.Formatter(
    fmt = '%(asctime)s [%(levelname)s] %(message)s',
    datefmt = '%m/%d/%Y %H:%M:%S'
  ))

  logger = stem.util.log.get_logger()
  logger.addHandler(debug_handler)

  nyxrc_content = "[file doesn't exist]"

  if os.path.exists(args.config):
    try:
      with open(args.config) as nyxrc_file:
        nyxrc_content = nyxrc_file.read()
    except IOError as exc:
      nyxrc_content = '[unable to read file: %s]' % exc.strerror

  log.trace(
    'debug.header',
    nyx_version = nyx.__version__,
    stem_version = stem.__version__,
    python_version = '.'.join(map(str, sys.version_info[:3])),
    system = platform.system(),
    platform = ' '.join(platform.dist()),
    nyxrc_path = args.config,
    nyxrc_content = nyxrc_content,
  )
Пример #12
0
def _setup_debug_logging(args):
  """
  Configures us to log at stem's trace level to a debug log path. This starts
  it off with some general diagnostic information.
  """

  debug_dir = os.path.dirname(args.debug_path)

  if not os.path.exists(debug_dir):
    os.makedirs(debug_dir)

  debug_handler = logging.FileHandler(args.debug_path, mode = 'w')
  debug_handler.setLevel(stem.util.log.logging_level(stem.util.log.TRACE))
  debug_handler.setFormatter(logging.Formatter(
    fmt = '%(asctime)s [%(levelname)s] %(message)s',
    datefmt = '%m/%d/%Y %H:%M:%S'
  ))

  logger = stem.util.log.get_logger()
  logger.addHandler(debug_handler)

  nyxrc_content = "[file doesn't exist]"

  if os.path.exists(args.config):
    try:
      with open(args.config) as nyxrc_file:
        nyxrc_content = nyxrc_file.read()
    except IOError as exc:
      nyxrc_content = '[unable to read file: %s]' % exc.strerror

  log.trace(
    'debug.header',
    nyx_version = nyx.__version__,
    stem_version = stem.__version__,
    python_version = '.'.join(map(str, sys.version_info[:3])),
    system = platform.system(),
    platform = ' '.join(platform.dist()),
    nyxrc_path = args.config,
    nyxrc_content = nyxrc_content,
  )
Пример #13
0
def call(command, default = UNDEFINED, ignore_exit_status = False):
  """
  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  :param str,list command: command to be issued
  :param object default: response if the query fails
  :param bool ignore_exit_status: reports failure if our command's exit status
    was non-zero

  :returns: **list** with the lines of output from the command

  :raises: **OSError** if this fails and no default was provided
  """

  if isinstance(command, str):
    command_list = command.split(' ')
  else:
    command_list = command

  try:
    is_shell_command = command_list[0] in SHELL_COMMANDS

    start_time = time.time()
    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command)

    stdout, stderr = process.communicate()
    stdout, stderr = stdout.strip(), stderr.strip()
    runtime = time.time() - start_time

    log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
    trace_prefix = 'Received from system (%s)' % command

    if stdout and stderr:
      log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
    elif stdout:
      log.trace(trace_prefix + ', stdout:\n%s' % stdout)
    elif stderr:
      log.trace(trace_prefix + ', stderr:\n%s' % stderr)

    exit_code = process.poll()

    if not ignore_exit_status and exit_code != 0:
      raise OSError('%s returned exit status %i' % (command, exit_code))

    if stdout:
      return stdout.decode('utf-8', 'replace').splitlines()
    else:
      return []
  except OSError as exc:
    log.debug('System call (failed): %s (error: %s)' % (command, exc))

    if default != UNDEFINED:
      return default
    else:
      raise exc
Пример #14
0
def call(command, default=UNDEFINED, ignore_exit_status=False):
    """
  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  :param str command: command to be issued
  :param object default: response if the query fails
  :param bool ignore_exit_status: reports failure if our command's exit status
    was non-zero

  :returns: **list** with the lines of output from the command

  :raises: **OSError** if this fails and no default was provided
  """

    try:
        is_shell_command = command.split(" ")[0] in SHELL_COMMANDS

        start_time = time.time()
        process = subprocess.Popen(command.split(),
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=is_shell_command)

        stdout, stderr = process.communicate()
        stdout, stderr = stdout.strip(), stderr.strip()
        runtime = time.time() - start_time

        log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
        trace_prefix = "Received from system (%s)" % command

        if stdout and stderr:
            log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" %
                      (stdout, stderr))
        elif stdout:
            log.trace(trace_prefix + ", stdout:\n%s" % stdout)
        elif stderr:
            log.trace(trace_prefix + ", stderr:\n%s" % stderr)

        exit_code = process.poll()

        if not ignore_exit_status and exit_code != 0:
            raise OSError("%s returned exit status %i" % (command, exit_code))

        if stdout:
            return stdout.decode("utf-8", "replace").splitlines()
        else:
            return []
    except OSError as exc:
        log.debug("System call (failed): %s (error: %s)" % (command, exc))

        if default != UNDEFINED:
            return default
        else:
            raise exc
Пример #15
0
def call(command, default = UNDEFINED):
  """
  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  :param str command: command to be issued
  :param object default: response if the query fails

  :returns: **list** with the lines of output from the command

  :raises: **OSError** if this fails and no default was provided
  """

  try:
    is_shell_command = command.split(" ")[0] in SHELL_COMMANDS

    start_time = time.time()
    stdout, stderr = subprocess.Popen(command.split(), stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command).communicate()
    stdout, stderr = stdout.strip(), stderr.strip()
    runtime = time.time() - start_time

    log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
    trace_prefix = "Received from system (%s)" % command

    if stdout and stderr:
      log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" % (stdout, stderr))
    elif stdout:
      log.trace(trace_prefix + ", stdout:\n%s" % stdout)
    elif stderr:
      log.trace(trace_prefix + ", stderr:\n%s" % stderr)

    if stdout:
      return stdout.decode("utf-8", "replace").splitlines()
    else:
      return []
  except OSError, exc:
    log.debug("System call (failed): %s (error: %s)" % (command, exc))

    if default != UNDEFINED:
      return default
    else:
      raise exc
Пример #16
0
def call(command, suppress_exc = True):
  """
  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.
  
  :param str command: command to be issued
  :param bool suppress_exc: if **True** then **None** is returned on failure,
    otherwise this raises the exception
  
  :returns: **list** with the lines of output from the command, **None** in
    case of failure if suppress_exc is **True**
  
  :raises: **OSError** if this fails and suppress_exc is **False**
  """
  
  try:
    start_time = time.time()
    stdout, stderr = subprocess.Popen(command.split(), stdout = subprocess.PIPE, stderr = subprocess.PIPE).communicate()
    stdout, stderr = stdout.strip(), stderr.strip()
    runtime = time.time() - start_time
    
    log.debug("System call: %s (runtime: %0.2f)" % (command, runtime))
    trace_prefix = "Received from system (%s)" % command
    
    if stdout and stderr:
      log.trace(trace_prefix + ", stdout:\n%s\nstderr:\n%s" % (stdout, stderr))
    elif stdout:
      log.trace(trace_prefix + ", stdout:\n%s" % stdout)
    elif stderr:
      log.trace(trace_prefix + ", stderr:\n%s" % stderr)
    
    if stdout: return stdout.splitlines()
    else: return []
  except OSError, exc:
    log.debug("System call (failed): %s (error: %s)" % (command, exc))
    
    if suppress_exc: return None
    else: raise exc
Пример #17
0
def recv_message(control_file):
    """
  Pulls from a control socket until we either have a complete message or
  encounter a problem.

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)

  :returns: :class:`~stem.response.ControlMessage` read from the socket

  :raises:
    * :class:`stem.ProtocolError` the content from the socket is malformed
    * :class:`stem.SocketClosed` if the socket closes before we receive
      a complete message
  """

    parsed_content, raw_content = [], ""
    logging_prefix = "Error while receiving a control message (%s): "

    while True:
        try:
            line = control_file.readline()
        except AttributeError:
            # if the control_file has been closed then we will receive:
            # AttributeError: 'NoneType' object has no attribute 'recv'

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "socket file has been closed")
            raise stem.SocketClosed("socket file has been closed")
        except socket.error, exc:
            # when disconnected we get...
            # socket.error: [Errno 107] Transport endpoint is not connected

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "received exception \"%s\"" % exc)
            raise stem.SocketClosed(exc)

        raw_content += line

        # Parses the tor control lines. These are of the form...
        # <status code><divider><content>\r\n

        if len(line) == 0:
            # if the socket is disconnected then the readline() method will provide
            # empty content

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "empty socket content")
            raise stem.SocketClosed("Received empty socket content.")
        elif len(line) < 4:
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "line too short, \"%s\"" % log.escape(line))
            raise stem.ProtocolError("Badly formatted reply line: too short")
        elif not re.match(r'^[a-zA-Z0-9]{3}[-+ ]', line):
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "malformed status code/divider, \"%s\"" %
                     log.escape(line))
            raise stem.ProtocolError(
                "Badly formatted reply line: beginning is malformed")
        elif not line.endswith("\r\n"):
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
            raise stem.ProtocolError("All lines should end with CRLF")

        line = line[:-2]  # strips off the CRLF
        status_code, divider, content = line[:3], line[3], line[4:]

        if divider == "-":
            # mid-reply line, keep pulling for more content
            parsed_content.append((status_code, divider, content))
        elif divider == " ":
            # end of the message, return the message
            parsed_content.append((status_code, divider, content))

            log_message = raw_content.replace("\r\n", "\n").rstrip()
            log.trace("Received from tor:\n" + log_message)

            return stem.response.ControlMessage(parsed_content, raw_content)
        elif divider == "+":
            # data entry, all of the following lines belong to the content until we
            # get a line with just a period

            while True:
                try:
                    line = control_file.readline()
                except socket.error, exc:
                    prefix = logging_prefix % "SocketClosed"
                    log.info(
                        prefix +
                        "received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")"
                        % (exc, log.escape(raw_content)))
                    raise stem.SocketClosed(exc)

                raw_content += line

                if not line.endswith("\r\n"):
                    prefix = logging_prefix % "ProtocolError"
                    log.info(
                        prefix +
                        "CRLF linebreaks missing from a data reply, \"%s\"" %
                        log.escape(raw_content))
                    raise stem.ProtocolError("All lines should end with CRLF")
                elif line == ".\r\n":
                    break  # data block termination

                line = line[:-2]  # strips off the CRLF

                # lines starting with a period are escaped by a second period (as per
                # section 2.4 of the control-spec)

                if line.startswith(".."):
                    line = line[1:]

                # appends to previous content, using a newline rather than CRLF
                # separator (more conventional for multi-line string content outside
                # the windows world)

                content += "\n" + line

            parsed_content.append((status_code, divider, content))
Пример #18
0
def recv_message(control_file):
    """
  Pulls from a control socket until we either have a complete message or
  encounter a problem.

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)

  :returns: :class:`~stem.response.ControlMessage` read from the socket

  :raises:
    * :class:`stem.ProtocolError` the content from the socket is malformed
    * :class:`stem.SocketClosed` if the socket closes before we receive
      a complete message
  """

    parsed_content, raw_content = [], b""
    logging_prefix = "Error while receiving a control message (%s): "

    while True:
        try:
            # From a real socket readline() would always provide bytes, but during
            # tests we might be given a StringIO in which case it's unicode under
            # python 3.x.

            line = stem.util.str_tools._to_bytes(control_file.readline())
        except AttributeError:
            # if the control_file has been closed then we will receive:
            # AttributeError: 'NoneType' object has no attribute 'recv'

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "socket file has been closed")
            raise stem.SocketClosed("socket file has been closed")
        except (socket.error, ValueError) as exc:
            # When disconnected we get...
            #
            # Python 2:
            #   socket.error: [Errno 107] Transport endpoint is not connected
            #
            # Python 3:
            #   ValueError: I/O operation on closed file.

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "received exception \"%s\"" % exc)
            raise stem.SocketClosed(exc)

        raw_content += line

        # Parses the tor control lines. These are of the form...
        # <status code><divider><content>\r\n

        if len(line) == 0:
            # if the socket is disconnected then the readline() method will provide
            # empty content

            prefix = logging_prefix % "SocketClosed"
            log.info(prefix + "empty socket content")
            raise stem.SocketClosed("Received empty socket content.")
        elif len(line) < 4:
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "line too short, \"%s\"" % log.escape(line))
            raise stem.ProtocolError("Badly formatted reply line: too short")
        elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "malformed status code/divider, \"%s\"" %
                     log.escape(line))
            raise stem.ProtocolError(
                "Badly formatted reply line: beginning is malformed")
        elif not line.endswith(b"\r\n"):
            prefix = logging_prefix % "ProtocolError"
            log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
            raise stem.ProtocolError("All lines should end with CRLF")

        line = line[:-2]  # strips off the CRLF
        status_code, divider, content = line[:3], line[3:4], line[4:]

        if stem.prereq.is_python_3():
            status_code = stem.util.str_tools._to_unicode(status_code)
            divider = stem.util.str_tools._to_unicode(divider)

        if divider == "-":
            # mid-reply line, keep pulling for more content
            parsed_content.append((status_code, divider, content))
        elif divider == " ":
            # end of the message, return the message
            parsed_content.append((status_code, divider, content))

            log_message = raw_content.replace(b"\r\n", b"\n").rstrip()
            log.trace("Received from tor:\n" +
                      stem.util.str_tools._to_unicode(log_message))

            return stem.response.ControlMessage(parsed_content, raw_content)
        elif divider == "+":
            # data entry, all of the following lines belong to the content until we
            # get a line with just a period

            while True:
                try:
                    line = stem.util.str_tools._to_bytes(
                        control_file.readline())
                except socket.error as exc:
                    prefix = logging_prefix % "SocketClosed"
                    log.info(
                        prefix +
                        "received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")"
                        % (exc, log.escape(raw_content)))
                    raise stem.SocketClosed(exc)

                raw_content += line

                if not line.endswith(b"\r\n"):
                    prefix = logging_prefix % "ProtocolError"
                    log.info(
                        prefix +
                        "CRLF linebreaks missing from a data reply, \"%s\"" %
                        log.escape(raw_content))
                    raise stem.ProtocolError("All lines should end with CRLF")
                elif line == b".\r\n":
                    break  # data block termination

                line = line[:-2]  # strips off the CRLF

                # lines starting with a period are escaped by a second period (as per
                # section 2.4 of the control-spec)

                if line.startswith(b".."):
                    line = line[1:]

                # appends to previous content, using a newline rather than CRLF
                # separator (more conventional for multi-line string content outside
                # the windows world)

                content += b"\n" + line

            parsed_content.append((status_code, divider, content))
        else:
            # this should never be reached due to the prefix regex, but might as well
            # be safe...
            prefix = logging_prefix % "ProtocolError"
            log.warn(prefix +
                     "\"%s\" isn't a recognized divider type" % divider)
            raise stem.ProtocolError(
                "Unrecognized divider type '%s': %s" %
                (divider, stem.util.str_tools._to_unicode(line)))
Пример #19
0
def call(command, default=UNDEFINED, ignore_exit_status=False, env=None):
    """
  call(command, default = UNDEFINED, ignore_exit_status = False)

  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  .. versionchanged:: 1.5.0
     Providing additional information upon failure by raising a CallError. This
     is a subclass of OSError, providing backward compatibility.

  .. versionchanged:: 1.5.0
     Added env argument.

  :param str,list command: command to be issued
  :param object default: response if the query fails
  :param bool ignore_exit_status: reports failure if our command's exit status
    was non-zero
  :param dict env: environment variables

  :returns: **list** with the lines of output from the command

  :raises: **CallError** if this fails and no default was provided, this is an
    **OSError** subclass
  """

    global SYSTEM_CALL_TIME

    if isinstance(command, str):
        command_list = command.split(' ')
    else:
        command_list = command

    exit_status, runtime, stdout, stderr = None, None, None, None
    start_time = time.time()

    try:
        is_shell_command = command_list[0] in SHELL_COMMANDS

        process = subprocess.Popen(command_list,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=is_shell_command,
                                   env=env)

        stdout, stderr = process.communicate()
        stdout, stderr = stdout.strip(), stderr.strip()
        runtime = time.time() - start_time

        log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
        trace_prefix = 'Received from system (%s)' % command

        if stdout and stderr:
            log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' %
                      (stdout, stderr))
        elif stdout:
            log.trace(trace_prefix + ', stdout:\n%s' % stdout)
        elif stderr:
            log.trace(trace_prefix + ', stderr:\n%s' % stderr)

        exit_status = process.poll()

        if not ignore_exit_status and exit_status != 0:
            raise OSError('%s returned exit status %i' %
                          (command, exit_status))

        if stdout:
            return stdout.decode('utf-8', 'replace').splitlines()
        else:
            return []
    except OSError as exc:
        log.debug('System call (failed): %s (error: %s)' % (command, exc))

        if default != UNDEFINED:
            return default
        else:
            raise CallError(str(exc), ' '.join(command_list), exit_status,
                            runtime, stdout, stderr)
    finally:
        with SYSTEM_CALL_TIME_LOCK:
            SYSTEM_CALL_TIME += time.time() - start_time
Пример #20
0
    def get_info(self,
                 params,
                 default=stem.control.UNDEFINED,
                 get_bytes=False,
                 cache_miss_warning=True):
        """
        get_info(params, default = UNDEFINED, get_bytes = False)

        Queries the control socket for the given GETINFO option. If provided a
        default then that's returned if the GETINFO option is undefined or the
        call fails for any reason (error response, control port closed, initiated,
        etc).

        .. versionchanged:: 1.1.0
           Added the get_bytes argument.

        :param str,list params: GETINFO option or options to be queried
        :param object default: response if the query fails
        :param bool get_bytes: provides **bytes** values rather than a **str** under python 3.x
        :param bool cache_miss_warning: Emit a warning if a cache miss happens

        :returns:
          Response depends upon how we were called as follows...

          * **str** with the response if our param was a **str**
          * **dict** with the 'param => response' mapping if our param was a **list**
          * default if one was provided and our call failed

        :raises:
          * :class:`stem.ControllerError` if the call fails and we weren't
            provided a default response
          * :class:`stem.InvalidArguments` if the 'params' requested was
            invalid
          * :class:`stem.ProtocolError` if the geoip database is known to be
            unavailable
        """

        start_time = time.time()
        reply = {}

        if isinstance(params, (bytes, str_type)):
            is_multiple = False
            params = set([params])
        else:
            if not params:
                return {}

            is_multiple = True
            params = set(params)

        # check for cached results

        from_cache = [param.lower() for param in params]
        cached_results = self._get_cache_map(from_cache, 'getinfo')

        for key in cached_results:
            user_expected_key = stem.control._case_insensitive_lookup(
                params, key)
            reply[user_expected_key] = cached_results[key]
            params.remove(user_expected_key)

        for param in params:
            if param.startswith(
                    'ip-to-country/') and self.is_geoip_unavailable():
                # the geoip database already looks to be unavailable - abort the request

                raise stem.ProtocolError('Tor geoip database is unavailable')

        # if everything was cached then short circuit making the query
        if not params:
            if stem.control.LOG_CACHE_FETCHES:
                log.trace('GETINFO %s (cache fetch)' % ' '.join(reply.keys()))

            if is_multiple:
                return reply
            else:
                return list(reply.values())[0]

        if cache_miss_warning is True:
            # As this should only happen when we intentionally refresh the cache (when the warning should be disabled)
            # we warn rather than debug to ensure we get the issue!
            lgr = logging.getLogger('theonionbox')
            lgr.info(
                'Cache miss for the following parameter(s): {}'.format(params))

        try:
            response = self.msg('GETINFO %s' % ' '.join(params))
            stem.response.convert('GETINFO', response)
            response._assert_matches(params)

            # usually we want unicode values under python 3.x

            if stem.prereq.is_python_3() and not get_bytes:
                response.entries = dict((k, stem.util.str_tools._to_unicode(v))
                                        for (k, v) in response.entries.items())

            reply.update(response.entries)

            if self.is_caching_enabled():
                to_cache = {}

                for key, value in response.entries.items():
                    key = key.lower()  # make case insensitive

                    # To allow The Onion Box smooth response cycles even when connecting to a relay / bridge
                    # via Tor socks proxy / hidden service, we cache *all* parameters and 'manually' update them with a
                    # single call every once in a while.

                    to_cache[key] = value
                    if key.startswith('ip-to-country/'):
                        # both cache-able and means that we should reset the geoip failure count
                        self._geoip_failure_count = -1

                self._set_cache(to_cache, 'getinfo')

            log.debug('GETINFO %s (runtime: %0.4f)' %
                      (' '.join(params), time.time() - start_time))

            if is_multiple:
                return reply
            else:
                return list(reply.values())[0]
        except stem.ControllerError as exc:
            # bump geoip failure count if...
            # * we're caching results
            # * this was soley a geoip lookup
            # * we've never had a successful geoip lookup (failure count isn't -1)

            is_geoip_request = len(params) == 1 and list(params)[0].startswith(
                'ip-to-country/')

            if is_geoip_request and self.is_caching_enabled(
            ) and self._geoip_failure_count != -1:
                self._geoip_failure_count += 1

                if self.is_geoip_unavailable():
                    log.warn("Tor's geoip database is unavailable.")

            log.debug('GETINFO %s (failed: %s)' % (' '.join(params), exc))

            raise
Пример #21
0
 def run(self):
   while not self._halt:
     minWait = self.resolveRate if self.resolveRate else self.defaultRate
     timeSinceReset = time.time() - self.lastLookup
     
     if self._isPaused or timeSinceReset < minWait:
       sleepTime = max(0.2, minWait - timeSinceReset)
       
       self._cond.acquire()
       if not self._halt: self._cond.wait(sleepTime)
       self._cond.release()
       
       continue # done waiting, try again
     
     isDefault = self.overwriteResolver == None
     resolver = self.defaultResolver if isDefault else self.overwriteResolver
     
     # checks if there's nothing to resolve with
     if not resolver:
       self.lastLookup = time.time() # avoids a busy wait in this case
       continue
     
     try:
       resolveStart = time.time()
       connResults = getConnections(resolver, self.processName, self.processPid)
       lookupTime = time.time() - resolveStart
       
       self._connections = connResults
       self._resolutionCounter += 1
       
       newMinDefaultRate = 100 * lookupTime
       if self.defaultRate < newMinDefaultRate:
         if self._rateThresholdBroken >= 3:
           # adding extra to keep the rate from frequently changing
           self.defaultRate = newMinDefaultRate + 0.5
           
           log.trace("connection lookup time increasing to %0.1f seconds per call" % self.defaultRate)
         else: self._rateThresholdBroken += 1
       else: self._rateThresholdBroken = 0
       
       if isDefault: self._subsiquentFailures = 0
     except (ValueError, IOError), exc:
       # this logs in a couple of cases:
       # - special failures noted by getConnections (most cases are already
       # logged via system)
       # - note fail-overs for default resolution methods
       if str(exc).startswith("No results found using:"):
         log.info(exc)
       
       if isDefault:
         self._subsiquentFailures += 1
         
         if self._subsiquentFailures >= RESOLVER_FAILURE_TOLERANCE:
           # failed several times in a row - abandon resolver and move on to another
           self._resolverBlacklist.append(resolver)
           self._subsiquentFailures = 0
           
           # pick another (non-blacklisted) resolver
           newResolver = None
           for r in self.resolverOptions:
             if not r in self._resolverBlacklist:
               newResolver = r
               break
           
           if newResolver:
             # provide notice that failures have occurred and resolver is changing
             log.notice(RESOLVER_SERIAL_FAILURE_MSG % (resolver, newResolver))
           else:
             # exhausted all resolvers, give warning
             log.notice(RESOLVER_FINAL_FAILURE_MSG)
           
           self.defaultResolver = newResolver
     finally:
Пример #22
0
def recv_message(control_file):
    """
  Pulls from a control socket until we either have a complete message or
  encounter a problem.

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)

  :returns: :class:`~stem.response.ControlMessage` read from the socket

  :raises:
    * :class:`stem.ProtocolError` the content from the socket is malformed
    * :class:`stem.SocketClosed` if the socket closes before we receive
      a complete message
  """

    parsed_content, raw_content = [], []
    logging_prefix = 'Error while receiving a control message (%s): '

    while True:
        try:
            # From a real socket readline() would always provide bytes, but during
            # tests we might be given a StringIO in which case it's unicode under
            # python 3.x.

            line = stem.util.str_tools._to_bytes(control_file.readline())
        except AttributeError:
            # if the control_file has been closed then we will receive:
            # AttributeError: 'NoneType' object has no attribute 'recv'

            prefix = logging_prefix % 'SocketClosed'
            log.info(prefix + 'socket file has been closed')
            raise stem.SocketClosed('socket file has been closed')
        except (socket.error, ValueError) as exc:
            # When disconnected we get...
            #
            # Python 2:
            #   socket.error: [Errno 107] Transport endpoint is not connected
            #
            # Python 3:
            #   ValueError: I/O operation on closed file.

            prefix = logging_prefix % 'SocketClosed'
            log.info(prefix + 'received exception "%s"' % exc)
            raise stem.SocketClosed(exc)

        raw_content.append(line)

        # Parses the tor control lines. These are of the form...
        # <status code><divider><content>\r\n

        if len(line) == 0:
            # if the socket is disconnected then the readline() method will provide
            # empty content

            prefix = logging_prefix % 'SocketClosed'
            log.info(prefix + 'empty socket content')
            raise stem.SocketClosed('Received empty socket content.')
        elif len(line) < 4:
            prefix = logging_prefix % 'ProtocolError'
            log.info(prefix + 'line too short, "%s"' % log.escape(line))
            raise stem.ProtocolError('Badly formatted reply line: too short')
        elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
            prefix = logging_prefix % 'ProtocolError'
            log.info(prefix +
                     'malformed status code/divider, "%s"' % log.escape(line))
            raise stem.ProtocolError(
                'Badly formatted reply line: beginning is malformed')
        elif not line.endswith(b'\r\n'):
            prefix = logging_prefix % 'ProtocolError'
            log.info(prefix + 'no CRLF linebreak, "%s"' % log.escape(line))
            raise stem.ProtocolError('All lines should end with CRLF')

        line = line[:-2]  # strips off the CRLF
        status_code, divider, content = line[:3], line[3:4], line[4:]
        content_lines = [content]

        if stem.prereq.is_python_3():
            status_code = stem.util.str_tools._to_unicode(status_code)
            divider = stem.util.str_tools._to_unicode(divider)

        if divider == '-':
            # mid-reply line, keep pulling for more content
            parsed_content.append((status_code, divider, content))
        elif divider == ' ':
            # end of the message, return the message
            parsed_content.append((status_code, divider, content))

            raw_content_str = b''.join(raw_content)
            log_message = stem.util.str_tools._to_unicode(
                raw_content_str.replace(b'\r\n', b'\n').rstrip())
            log_message_lines = log_message.split('\n')

            if TRUNCATE_LOGS and len(log_message_lines) > TRUNCATE_LOGS:
                log_message = '\n'.join(log_message_lines[:TRUNCATE_LOGS] + [
                    '... %i more lines...' %
                    (len(log_message_lines) - TRUNCATE_LOGS)
                ])

            if len(log_message_lines) > 2:
                log.trace('Received from tor:\n%s' % log_message)
            else:
                log.trace('Received from tor: %s' %
                          log_message.replace('\n', '\\n'))

            return stem.response.ControlMessage(parsed_content,
                                                raw_content_str)
        elif divider == '+':
            # data entry, all of the following lines belong to the content until we
            # get a line with just a period

            while True:
                try:
                    line = stem.util.str_tools._to_bytes(
                        control_file.readline())
                except socket.error as exc:
                    prefix = logging_prefix % 'SocketClosed'
                    log.info(
                        prefix +
                        'received an exception while mid-way through a data reply (exception: "%s", read content: "%s")'
                        % (exc, log.escape(b''.join(raw_content))))
                    raise stem.SocketClosed(exc)

                raw_content.append(line)

                if not line.endswith(b'\r\n'):
                    prefix = logging_prefix % 'ProtocolError'
                    log.info(
                        prefix +
                        'CRLF linebreaks missing from a data reply, "%s"' %
                        log.escape(b''.join(raw_content)))
                    raise stem.ProtocolError('All lines should end with CRLF')
                elif line == b'.\r\n':
                    break  # data block termination

                line = line[:-2]  # strips off the CRLF

                # lines starting with a period are escaped by a second period (as per
                # section 2.4 of the control-spec)

                if line.startswith(b'..'):
                    line = line[1:]

                content_lines.append(line)

            # joins the content using a newline rather than CRLF separator (more
            # conventional for multi-line string content outside the windows world)

            parsed_content.append(
                (status_code, divider, b'\n'.join(content_lines)))
        else:
            # this should never be reached due to the prefix regex, but might as well
            # be safe...
            prefix = logging_prefix % 'ProtocolError'
            log.warn(prefix +
                     "\"%s\" isn't a recognized divider type" % divider)
            raise stem.ProtocolError(
                "Unrecognized divider type '%s': %s" %
                (divider, stem.util.str_tools._to_unicode(line)))
Пример #23
0
def recv_message(control_file):
  """
  Pulls from a control socket until we either have a complete message or
  encounter a problem.

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)

  :returns: :class:`~stem.response.ControlMessage` read from the socket

  :raises:
    * :class:`stem.ProtocolError` the content from the socket is malformed
    * :class:`stem.SocketClosed` if the socket closes before we receive
      a complete message
  """

  parsed_content, raw_content = [], ""
  logging_prefix = "Error while receiving a control message (%s): "

  while True:
    try:
      line = control_file.readline()
    except AttributeError:
      # if the control_file has been closed then we will receive:
      # AttributeError: 'NoneType' object has no attribute 'recv'

      prefix = logging_prefix % "SocketClosed"
      log.info(prefix + "socket file has been closed")
      raise stem.SocketClosed("socket file has been closed")
    except socket.error, exc:
      # when disconnected we get...
      # socket.error: [Errno 107] Transport endpoint is not connected

      prefix = logging_prefix % "SocketClosed"
      log.info(prefix + "received exception \"%s\"" % exc)
      raise stem.SocketClosed(exc)

    raw_content += line

    # Parses the tor control lines. These are of the form...
    # <status code><divider><content>\r\n

    if len(line) == 0:
      # if the socket is disconnected then the readline() method will provide
      # empty content

      prefix = logging_prefix % "SocketClosed"
      log.info(prefix + "empty socket content")
      raise stem.SocketClosed("Received empty socket content.")
    elif len(line) < 4:
      prefix = logging_prefix % "ProtocolError"
      log.info(prefix + "line too short, \"%s\"" % log.escape(line))
      raise stem.ProtocolError("Badly formatted reply line: too short")
    elif not re.match(r'^[a-zA-Z0-9]{3}[-+ ]', line):
      prefix = logging_prefix % "ProtocolError"
      log.info(prefix + "malformed status code/divider, \"%s\"" % log.escape(line))
      raise stem.ProtocolError("Badly formatted reply line: beginning is malformed")
    elif not line.endswith("\r\n"):
      prefix = logging_prefix % "ProtocolError"
      log.info(prefix + "no CRLF linebreak, \"%s\"" % log.escape(line))
      raise stem.ProtocolError("All lines should end with CRLF")

    line = line[:-2]  # strips off the CRLF
    status_code, divider, content = line[:3], line[3], line[4:]

    if divider == "-":
      # mid-reply line, keep pulling for more content
      parsed_content.append((status_code, divider, content))
    elif divider == " ":
      # end of the message, return the message
      parsed_content.append((status_code, divider, content))

      log_message = raw_content.replace("\r\n", "\n").rstrip()
      log.trace("Received from tor:\n" + log_message)

      return stem.response.ControlMessage(parsed_content, raw_content)
    elif divider == "+":
      # data entry, all of the following lines belong to the content until we
      # get a line with just a period

      while True:
        try:
          line = control_file.readline()
        except socket.error, exc:
          prefix = logging_prefix % "SocketClosed"
          log.info(prefix + "received an exception while mid-way through a data reply (exception: \"%s\", read content: \"%s\")" % (exc, log.escape(raw_content)))
          raise stem.SocketClosed(exc)

        raw_content += line

        if not line.endswith("\r\n"):
          prefix = logging_prefix % "ProtocolError"
          log.info(prefix + "CRLF linebreaks missing from a data reply, \"%s\"" % log.escape(raw_content))
          raise stem.ProtocolError("All lines should end with CRLF")
        elif line == ".\r\n":
          break  # data block termination

        line = line[:-2]  # strips off the CRLF

        # lines starting with a period are escaped by a second period (as per
        # section 2.4 of the control-spec)

        if line.startswith(".."):
          line = line[1:]

        # appends to previous content, using a newline rather than CRLF
        # separator (more conventional for multi-line string content outside
        # the windows world)

        content += "\n" + line

      parsed_content.append((status_code, divider, content))
Пример #24
0
def recv_message(control_file):
  """
  Pulls from a control socket until we either have a complete message or
  encounter a problem.

  :param file control_file: file derived from the control socket (see the
    socket's makefile() method for more information)

  :returns: :class:`~stem.response.ControlMessage` read from the socket

  :raises:
    * :class:`stem.ProtocolError` the content from the socket is malformed
    * :class:`stem.SocketClosed` if the socket closes before we receive
      a complete message
  """

  parsed_content, raw_content = [], []
  logging_prefix = 'Error while receiving a control message (%s): '

  while True:
    try:
      # From a real socket readline() would always provide bytes, but during
      # tests we might be given a StringIO in which case it's unicode under
      # python 3.x.

      line = stem.util.str_tools._to_bytes(control_file.readline())
    except AttributeError:
      # if the control_file has been closed then we will receive:
      # AttributeError: 'NoneType' object has no attribute 'recv'

      prefix = logging_prefix % 'SocketClosed'
      log.info(prefix + 'socket file has been closed')
      raise stem.SocketClosed('socket file has been closed')
    except (socket.error, ValueError) as exc:
      # When disconnected we get...
      #
      # Python 2:
      #   socket.error: [Errno 107] Transport endpoint is not connected
      #
      # Python 3:
      #   ValueError: I/O operation on closed file.

      prefix = logging_prefix % 'SocketClosed'
      log.info(prefix + 'received exception "%s"' % exc)
      raise stem.SocketClosed(exc)

    raw_content.append(line)

    # Parses the tor control lines. These are of the form...
    # <status code><divider><content>\r\n

    if len(line) == 0:
      # if the socket is disconnected then the readline() method will provide
      # empty content

      prefix = logging_prefix % 'SocketClosed'
      log.info(prefix + 'empty socket content')
      raise stem.SocketClosed('Received empty socket content.')
    elif len(line) < 4:
      prefix = logging_prefix % 'ProtocolError'
      log.info(prefix + 'line too short, "%s"' % log.escape(line))
      raise stem.ProtocolError('Badly formatted reply line: too short')
    elif not re.match(b'^[a-zA-Z0-9]{3}[-+ ]', line):
      prefix = logging_prefix % 'ProtocolError'
      log.info(prefix + 'malformed status code/divider, "%s"' % log.escape(line))
      raise stem.ProtocolError('Badly formatted reply line: beginning is malformed')
    elif not line.endswith(b'\r\n'):
      prefix = logging_prefix % 'ProtocolError'
      log.info(prefix + 'no CRLF linebreak, "%s"' % log.escape(line))
      raise stem.ProtocolError('All lines should end with CRLF')

    line = line[:-2]  # strips off the CRLF
    status_code, divider, content = line[:3], line[3:4], line[4:]
    content_lines = [content]

    if stem.prereq.is_python_3():
      status_code = stem.util.str_tools._to_unicode(status_code)
      divider = stem.util.str_tools._to_unicode(divider)

    if divider == '-':
      # mid-reply line, keep pulling for more content
      parsed_content.append((status_code, divider, content))
    elif divider == ' ':
      # end of the message, return the message
      parsed_content.append((status_code, divider, content))

      raw_content_str = b''.join(raw_content)
      log_message = raw_content_str.replace(b'\r\n', b'\n').rstrip()
      log.trace('Received from tor:\n' + stem.util.str_tools._to_unicode(log_message))

      return stem.response.ControlMessage(parsed_content, raw_content_str)
    elif divider == '+':
      # data entry, all of the following lines belong to the content until we
      # get a line with just a period

      while True:
        try:
          line = stem.util.str_tools._to_bytes(control_file.readline())
        except socket.error as exc:
          prefix = logging_prefix % 'SocketClosed'
          log.info(prefix + 'received an exception while mid-way through a data reply (exception: "%s", read content: "%s")' % (exc, log.escape(b''.join(raw_content))))
          raise stem.SocketClosed(exc)

        raw_content.append(line)

        if not line.endswith(b'\r\n'):
          prefix = logging_prefix % 'ProtocolError'
          log.info(prefix + 'CRLF linebreaks missing from a data reply, "%s"' % log.escape(b''.join(raw_content)))
          raise stem.ProtocolError('All lines should end with CRLF')
        elif line == b'.\r\n':
          break  # data block termination

        line = line[:-2]  # strips off the CRLF

        # lines starting with a period are escaped by a second period (as per
        # section 2.4 of the control-spec)

        if line.startswith(b'..'):
          line = line[1:]

        content_lines.append(line)

      # joins the content using a newline rather than CRLF separator (more
      # conventional for multi-line string content outside the windows world)

      parsed_content.append((status_code, divider, b'\n'.join(content_lines)))
    else:
      # this should never be reached due to the prefix regex, but might as well
      # be safe...
      prefix = logging_prefix % 'ProtocolError'
      log.warn(prefix + "\"%s\" isn't a recognized divider type" % divider)
      raise stem.ProtocolError("Unrecognized divider type '%s': %s" % (divider, stem.util.str_tools._to_unicode(line)))
Пример #25
0
def call(command, default = UNDEFINED, ignore_exit_status = False, timeout = None, cwd = None, env = None):
  """
  call(command, default = UNDEFINED, ignore_exit_status = False)

  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  .. versionchanged:: 1.5.0
     Providing additional information upon failure by raising a CallError. This
     is a subclass of OSError, providing backward compatibility.

  .. versionchanged:: 1.5.0
     Added env argument.

  .. versionchanged:: 1.6.0
     Added timeout and cwd arguments.

  :param str,list command: command to be issued
  :param object default: response if the query fails
  :param bool ignore_exit_status: reports failure if our command's exit status
    was non-zero
  :param float timeout: maximum seconds to wait, blocks indefinitely if
    **None**
  :param dict env: environment variables

  :returns: **list** with the lines of output from the command

  :raises:
    * **CallError** if this fails and no default was provided
    * **CallTimeoutError** if the timeout is reached without a default
  """

  global SYSTEM_CALL_TIME

  if isinstance(command, str):
    command_list = command.split(' ')
  else:
    command_list = list(map(str, command))

  exit_status, runtime, stdout, stderr = None, None, None, None
  start_time = time.time()

  try:
    is_shell_command = command_list[0] in SHELL_COMMANDS

    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, cwd = cwd, env = env)

    if timeout:
      while process.poll() is None:
        if time.time() - start_time > timeout:
          raise CallTimeoutError("Process didn't finish after %0.1f seconds" % timeout, ' '.join(command_list), None, timeout, '', '', timeout)

        time.sleep(0.001)

    stdout, stderr = process.communicate()
    stdout, stderr = stdout.strip(), stderr.strip()
    runtime = time.time() - start_time

    log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
    trace_prefix = 'Received from system (%s)' % command

    if stdout and stderr:
      log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
    elif stdout:
      log.trace(trace_prefix + ', stdout:\n%s' % stdout)
    elif stderr:
      log.trace(trace_prefix + ', stderr:\n%s' % stderr)

    exit_status = process.poll()

    if not ignore_exit_status and exit_status != 0:
      raise OSError('%s returned exit status %i' % (command, exit_status))

    if stdout:
      return stdout.decode('utf-8', 'replace').splitlines()
    else:
      return []
  except CallTimeoutError as exc:
    log.debug('System call (timeout): %s (after %0.4fs)' % (command, timeout))

    if default != UNDEFINED:
      return default
    else:
      raise
  except OSError as exc:
    log.debug('System call (failed): %s (error: %s)' % (command, exc))

    if default != UNDEFINED:
      return default
    else:
      raise CallError(str(exc), ' '.join(command_list), exit_status, runtime, stdout, stderr)
  finally:
    with SYSTEM_CALL_TIME_LOCK:
      SYSTEM_CALL_TIME += time.time() - start_time
Пример #26
0
def call(command, default = UNDEFINED, ignore_exit_status = False, timeout = None, cwd = None, env = None):
  """
  call(command, default = UNDEFINED, ignore_exit_status = False)

  Issues a command in a subprocess, blocking until completion and returning the
  results. This is not actually ran in a shell so pipes and other shell syntax
  are not permitted.

  .. versionchanged:: 1.5.0
     Providing additional information upon failure by raising a CallError. This
     is a subclass of OSError, providing backward compatibility.

  .. versionchanged:: 1.5.0
     Added env argument.

  .. versionchanged:: 1.6.0
     Added timeout and cwd arguments.

  :param str,list command: command to be issued
  :param object default: response if the query fails
  :param bool ignore_exit_status: reports failure if our command's exit status
    was non-zero
  :param float timeout: maximum seconds to wait, blocks indefinitely if
    **None**
  :param dict env: environment variables

  :returns: **list** with the lines of output from the command

  :raises:
    * **CallError** if this fails and no default was provided
    * **CallTimeoutError** if the timeout is reached without a default
  """

  # TODO: in stem 2.x return a struct with stdout, stderr, and runtime instead

  global SYSTEM_CALL_TIME

  if isinstance(command, str):
    command_list = command.split(' ')
  else:
    command_list = list(map(str, command))

  exit_status, runtime, stdout, stderr = None, None, None, None
  start_time = time.time()

  try:
    is_shell_command = command_list[0] in SHELL_COMMANDS

    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, cwd = cwd, env = env)

    if timeout:
      while process.poll() is None:
        if time.time() - start_time > timeout:
          raise CallTimeoutError("Process didn't finish after %0.1f seconds" % timeout, ' '.join(command_list), None, timeout, '', '', timeout)

        time.sleep(0.001)

    stdout, stderr = process.communicate()
    stdout, stderr = stdout.strip(), stderr.strip()
    runtime = time.time() - start_time

    log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))

    if log.is_tracing():
      trace_prefix = 'Received from system (%s)' % command

      if stdout and stderr:
        log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
      elif stdout:
        log.trace(trace_prefix + ', stdout:\n%s' % stdout)
      elif stderr:
        log.trace(trace_prefix + ', stderr:\n%s' % stderr)

    exit_status = process.poll()

    if not ignore_exit_status and exit_status != 0:
      raise OSError('%s returned exit status %i' % (command, exit_status))

    if stdout:
      return stdout.decode('utf-8', 'replace').splitlines()
    else:
      return []
  except CallTimeoutError:
    log.debug('System call (timeout): %s (after %0.4fs)' % (command, timeout))

    if default != UNDEFINED:
      return default
    else:
      raise
  except OSError as exc:
    log.debug('System call (failed): %s (error: %s)' % (command, exc))

    if default != UNDEFINED:
      return default
    else:
      raise CallError(str(exc), ' '.join(command_list), exit_status, runtime, stdout, stderr)
  finally:
    with SYSTEM_CALL_TIME_LOCK:
      SYSTEM_CALL_TIME += time.time() - start_time