Example #1
0
  def __init__(self, raw_contents, validate = False):
    super(HiddenServiceDescriptor, self).__init__(raw_contents, lazy_load = not validate)
    entries = _get_descriptor_components(raw_contents, validate, non_ascii_fields = ('introduction-points'))

    if validate:
      for keyword in REQUIRED_FIELDS:
        if keyword not in entries:
          raise ValueError("Hidden service descriptor must have a '%s' entry" % keyword)
        elif keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("The '%s' entry can only appear once in a hidden service descriptor" % keyword)

      if 'rendezvous-service-descriptor' != list(entries.keys())[0]:
        raise ValueError("Hidden service descriptor must start with a 'rendezvous-service-descriptor' entry")
      elif 'signature' != list(entries.keys())[-1]:
        raise ValueError("Hidden service descriptor must end with a 'signature' entry")

      self._parse(entries, validate)

      if stem.prereq.is_crypto_available():
        signed_digest = self._digest_for_signature(self.permanent_key, self.signature)
        content_digest = self._digest_for_content(b'rendezvous-service-descriptor ', b'\nsignature\n')

        if signed_digest != content_digest:
          raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, content_digest))
    else:
      self._entries = entries
  def __init__(self, raw_contents, validate = False):
    super(HiddenServiceDescriptor, self).__init__(raw_contents, lazy_load = not validate)
    entries = _get_descriptor_components(raw_contents, validate, non_ascii_fields = ('introduction-points'))

    if validate:
      for keyword in REQUIRED_FIELDS:
        if keyword not in entries:
          raise ValueError("Hidden service descriptor must have a '%s' entry" % keyword)
        elif keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("The '%s' entry can only appear once in a hidden service descriptor" % keyword)

      if 'rendezvous-service-descriptor' != list(entries.keys())[0]:
        raise ValueError("Hidden service descriptor must start with a 'rendezvous-service-descriptor' entry")
      elif 'signature' != list(entries.keys())[-1]:
        raise ValueError("Hidden service descriptor must end with a 'signature' entry")

      self._parse(entries, validate)

      if stem.prereq.is_crypto_available():
        signed_digest = self._digest_for_signature(self.permanent_key, self.signature)
        content_digest = self._digest_for_content(b'rendezvous-service-descriptor ', b'\nsignature\n')

        if signed_digest != content_digest:
          raise ValueError('Decrypted digest does not match local digest (calculated: %s, local: %s)' % (signed_digest, content_digest))
    else:
      self._entries = entries
  def __init__(self, content, validate = False, document = None):
    """
    Parse a router descriptor in a network status document.

    :param str content: router descriptor content to be parsed
    :param NetworkStatusDocument document: document this descriptor came from
    :param bool validate: checks the validity of the content if **True**, skips
      these checks otherwise

    :raises: **ValueError** if the descriptor data is invalid
    """

    super(RouterStatusEntry, self).__init__(content, lazy_load = not validate)
    self.document = document
    entries = _get_descriptor_components(content, validate)

    if validate:
      for keyword in self._required_fields():
        if keyword not in entries:
          raise ValueError("%s must have a '%s' line:\n%s" % (self._name(True), keyword, str(self)))

      for keyword in self._single_fields():
        if keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("%s can only have a single '%s' line, got %i:\n%s" % (self._name(True), keyword, len(entries[keyword]), str(self)))

      if 'r' != list(entries.keys())[0]:
        raise ValueError("%s are expected to start with a 'r' line:\n%s" % (self._name(True), str(self)))

      self._parse(entries, validate)
    else:
      self._entries = entries
Example #4
0
  def __init__(self, content, validate = False, document = None):
    """
    Parse a router descriptor in a network status document.

    :param str content: router descriptor content to be parsed
    :param NetworkStatusDocument document: document this descriptor came from
    :param bool validate: checks the validity of the content if **True**, skips
      these checks otherwise

    :raises: **ValueError** if the descriptor data is invalid
    """

    super(RouterStatusEntry, self).__init__(content, lazy_load = not validate)
    self.document = document
    entries = _get_descriptor_components(content, validate)

    if validate:
      for keyword in self._required_fields():
        if keyword not in entries:
          raise ValueError("%s must have a '%s' line:\n%s" % (self._name(True), keyword, str(self)))

      for keyword in self._single_fields():
        if keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("%s can only have a single '%s' line, got %i:\n%s" % (self._name(True), keyword, len(entries[keyword]), str(self)))

      if 'r' != list(entries.keys())[0]:
        raise ValueError("%s are expected to start with a 'r' line:\n%s" % (self._name(True), str(self)))

      self._parse(entries, validate)
    else:
      self._entries = entries
Example #5
0
  def __init__(self, raw_contents, validate = False, annotations = None):
    super(Microdescriptor, self).__init__(raw_contents, lazy_load = not validate)
    self._annotation_lines = annotations if annotations else []
    entries = _get_descriptor_components(raw_contents, validate)

    if validate:
      self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper()
      self._parse(entries, validate)
      self._check_constraints(entries)
    else:
      self._entries = entries
Example #6
0
    def __init__(self, raw_contents, validate):
        super(TorDNSEL, self).__init__(raw_contents)
        raw_contents = stem.util.str_tools._to_unicode(raw_contents)
        entries = _get_descriptor_components(raw_contents, validate)

        self.fingerprint = None
        self.published = None
        self.last_status = None
        self.exit_addresses = []

        self._parse(entries, validate)
Example #7
0
    def __init__(self, raw_contents, validate=False, annotations=None):
        """
    Server descriptor constructor, created from an individual relay's
    descriptor content (as provided by 'GETINFO desc/*', cached descriptors,
    and metrics).

    By default this validates the descriptor's content as it's parsed. This
    validation can be disables to either improve performance or be accepting of
    malformed data.

    :param str raw_contents: descriptor content provided by the relay
    :param bool validate: checks the validity of the descriptor's content if
      **True**, skips these checks otherwise
    :param list annotations: lines that appeared prior to the descriptor

    :raises: **ValueError** if the contents is malformed and validate is True
    """

        super(ServerDescriptor, self).__init__(raw_contents,
                                               lazy_load=not validate)
        self._annotation_lines = annotations if annotations else []

        # A descriptor contains a series of 'keyword lines' which are simply a
        # keyword followed by an optional value. Lines can also be followed by a
        # signature block.
        #
        # We care about the ordering of 'accept' and 'reject' entries because this
        # influences the resulting exit policy, but for everything else the order
        # does not matter so breaking it into key / value pairs.

        entries, self._unparsed_exit_policy = _get_descriptor_components(
            stem.util.str_tools._to_unicode(raw_contents),
            validate,
            extra_keywords=('accept', 'reject'),
            non_ascii_fields=('contact', 'platform'))

        if validate:
            self._parse(entries, validate)

            _parse_exit_policy(self, entries)

            # if we have a negative uptime and a tor version that shouldn't exhibit
            # this bug then fail validation

            if validate and self.uptime and self.tor_version:
                if self.uptime < 0 and self.tor_version >= stem.version.Version(
                        '0.1.2.7'):
                    raise ValueError(
                        "Descriptor for version '%s' had a negative uptime value: %i"
                        % (self.tor_version, self.uptime))

            self._check_constraints(entries)
        else:
            self._entries = entries
  def __init__(self, raw_contents, validate):
    super(TorDNSEL, self).__init__(raw_contents)
    raw_contents = stem.util.str_tools._to_unicode(raw_contents)
    entries = _get_descriptor_components(raw_contents, validate)

    self.fingerprint = None
    self.published = None
    self.last_status = None
    self.exit_addresses = []

    self._parse(entries, validate)
Example #9
0
    def __init__(self, raw_contents, validate=False, annotations=None):
        super(Microdescriptor, self).__init__(raw_contents,
                                              lazy_load=not validate)
        self._annotation_lines = annotations if annotations else []
        entries = _get_descriptor_components(raw_contents, validate)

        if validate:
            self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper()
            self._parse(entries, validate)
            self._check_constraints(entries)
        else:
            self._entries = entries
  def _parse_introduction_points(content):
    """
    Provides the parsed list of IntroductionPoints for the unencrypted content.
    """

    introduction_points = []
    content_io = io.BytesIO(content)

    while True:
      content = b''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True))

      if not content:
        break  # reached the end

      attr = dict(INTRODUCTION_POINTS_ATTR)
      entries = _get_descriptor_components(content, False)

      for keyword, values in list(entries.items()):
        value, block_type, block_contents = values[0]

        if keyword in SINGLE_INTRODUCTION_POINT_FIELDS and len(values) > 1:
          raise ValueError("'%s' can only appear once in an introduction-point block, but appeared %i times" % (keyword, len(values)))

        if keyword == 'introduction-point':
          attr['identifier'] = value
        elif keyword == 'ip-address':
          if not stem.util.connection.is_valid_ipv4_address(value):
            raise ValueError("'%s' is an invalid IPv4 address" % value)

          attr['address'] = value
        elif keyword == 'onion-port':
          if not stem.util.connection.is_valid_port(value):
            raise ValueError("'%s' is an invalid port" % value)

          attr['port'] = int(value)
        elif keyword == 'onion-key':
          attr['onion_key'] = block_contents
        elif keyword == 'service-key':
          attr['service_key'] = block_contents
        elif keyword == 'intro-authentication':
          auth_entries = []

          for auth_value, _, _ in values:
            if ' ' not in auth_value:
              raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value)

            auth_type, auth_data = auth_value.split(' ')[:2]
            auth_entries.append((auth_type, auth_data))

      introduction_points.append(IntroductionPoints(**attr))

    return introduction_points
Example #11
0
  def _parse_introduction_points(content):
    """
    Provides the parsed list of IntroductionPoints for the unencrypted content.
    """

    introduction_points = []
    content_io = io.BytesIO(content)

    while True:
      content = b''.join(_read_until_keywords('introduction-point', content_io, ignore_first = True))

      if not content:
        break  # reached the end

      attr = dict(INTRODUCTION_POINTS_ATTR)
      entries = _get_descriptor_components(content, False)

      for keyword, values in list(entries.items()):
        value, block_type, block_contents = values[0]

        if keyword in SINGLE_INTRODUCTION_POINT_FIELDS and len(values) > 1:
          raise ValueError("'%s' can only appear once in an introduction-point block, but appeared %i times" % (keyword, len(values)))

        if keyword == 'introduction-point':
          attr['identifier'] = value
        elif keyword == 'ip-address':
          if not stem.util.connection.is_valid_ipv4_address(value):
            raise ValueError("'%s' is an invalid IPv4 address" % value)

          attr['address'] = value
        elif keyword == 'onion-port':
          if not stem.util.connection.is_valid_port(value):
            raise ValueError("'%s' is an invalid port" % value)

          attr['port'] = int(value)
        elif keyword == 'onion-key':
          attr['onion_key'] = block_contents
        elif keyword == 'service-key':
          attr['service_key'] = block_contents
        elif keyword == 'intro-authentication':
          auth_entries = []

          for auth_value, _, _ in values:
            if ' ' not in auth_value:
              raise ValueError("We expected 'intro-authentication [auth_type] [auth_data]', but had '%s'" % auth_value)

            auth_type, auth_data = auth_value.split(' ')[:2]
            auth_entries.append((auth_type, auth_data))

      introduction_points.append(IntroductionPoints(**attr))

    return introduction_points
Example #12
0
  def __init__(self, raw_contents, validate = False, annotations = None):
    """
    Server descriptor constructor, created from an individual relay's
    descriptor content (as provided by 'GETINFO desc/*', cached descriptors,
    and metrics).

    By default this validates the descriptor's content as it's parsed. This
    validation can be disables to either improve performance or be accepting of
    malformed data.

    :param str raw_contents: descriptor content provided by the relay
    :param bool validate: checks the validity of the descriptor's content if
      **True**, skips these checks otherwise
    :param list annotations: lines that appeared prior to the descriptor

    :raises: **ValueError** if the contents is malformed and validate is True
    """

    super(ServerDescriptor, self).__init__(raw_contents, lazy_load = not validate)
    self._annotation_lines = annotations if annotations else []

    # A descriptor contains a series of 'keyword lines' which are simply a
    # keyword followed by an optional value. Lines can also be followed by a
    # signature block.
    #
    # We care about the ordering of 'accept' and 'reject' entries because this
    # influences the resulting exit policy, but for everything else the order
    # does not matter so breaking it into key / value pairs.

    entries, self._unparsed_exit_policy = _get_descriptor_components(stem.util.str_tools._to_unicode(raw_contents), validate, extra_keywords = ('accept', 'reject'), non_ascii_fields = ('contact', 'platform'))

    if validate:
      self._parse(entries, validate)

      _parse_exit_policy(self, entries)

      # if we have a negative uptime and a tor version that shouldn't exhibit
      # this bug then fail validation

      if validate and self.uptime and self.tor_version:
        if self.uptime < 0 and self.tor_version >= stem.version.Version('0.1.2.7'):
          raise ValueError("Descriptor for version '%s' had a negative uptime value: %i" % (self.tor_version, self.uptime))

      self._check_constraints(entries)
    else:
      self._entries = entries
Example #13
0
    def __init__(self, raw_contents, validate=False):
        """
    Extra-info descriptor constructor. By default this validates the
    descriptor's content as it's parsed. This validation can be disabled to
    either improve performance or be accepting of malformed data.

    :param str raw_contents: extra-info content provided by the relay
    :param bool validate: checks the validity of the extra-info descriptor if
      **True**, skips these checks otherwise

    :raises: **ValueError** if the contents is malformed and validate is True
    """

        super(ExtraInfoDescriptor, self).__init__(raw_contents,
                                                  lazy_load=not validate)
        entries = _get_descriptor_components(raw_contents, validate)

        if validate:
            for keyword in self._required_fields():
                if keyword not in entries:
                    raise ValueError(
                        "Extra-info descriptor must have a '%s' entry" %
                        keyword)

            for keyword in self._required_fields() + SINGLE_FIELDS:
                if keyword in entries and len(entries[keyword]) > 1:
                    raise ValueError(
                        "The '%s' entry can only appear once in an extra-info descriptor"
                        % keyword)

            expected_first_keyword = self._first_keyword()
            if expected_first_keyword and expected_first_keyword != list(
                    entries.keys())[0]:
                raise ValueError(
                    "Extra-info descriptor must start with a '%s' entry" %
                    expected_first_keyword)

            expected_last_keyword = self._last_keyword()
            if expected_last_keyword and expected_last_keyword != list(
                    entries.keys())[-1]:
                raise ValueError("Descriptor must end with a '%s' entry" %
                                 expected_last_keyword)

            self._parse(entries, validate)
        else:
            self._entries = entries
Example #14
0
  def __init__(self, raw_contents, validate = True, annotations = None):
    super(Microdescriptor, self).__init__(raw_contents)
    raw_contents = stem.util.str_tools._to_unicode(raw_contents)

    self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper()

    self.onion_key = None
    self.ntor_onion_key = None
    self.or_addresses = []
    self.family = []
    self.exit_policy = stem.exit_policy.MicroExitPolicy("reject 1-65535")
    self.exit_policy_v6 = None

    self._unrecognized_lines = []

    self._annotation_lines = annotations if annotations else []

    entries = _get_descriptor_components(raw_contents, validate)
    self._parse(entries, validate)

    if validate:
      self._check_constraints(entries)
Example #15
0
    def __init__(self, raw_contents, validate=True, annotations=None):
        super(Microdescriptor, self).__init__(raw_contents)
        raw_contents = stem.util.str_tools._to_unicode(raw_contents)

        self.digest = hashlib.sha256(self.get_bytes()).hexdigest().upper()

        self.onion_key = None
        self.ntor_onion_key = None
        self.or_addresses = []
        self.family = []
        self.exit_policy = stem.exit_policy.MicroExitPolicy("reject 1-65535")
        self.exit_policy_v6 = None

        self._unrecognized_lines = []

        self._annotation_lines = annotations if annotations else []

        entries = _get_descriptor_components(raw_contents, validate)
        self._parse(entries, validate)

        if validate:
            self._check_constraints(entries)
Example #16
0
  def __init__(self, content, validate, document):
    """
    Parse a router descriptor in a network status document.

    :param str content: router descriptor content to be parsed
    :param NetworkStatusDocument document: document this descriptor came from
    :param bool validate: checks the validity of the content if **True**, skips
      these checks otherwise

    :raises: **ValueError** if the descriptor data is invalid
    """

    super(RouterStatusEntry, self).__init__(content)
    content = stem.util.str_tools._to_unicode(content)

    self.document = document

    self.nickname = None
    self.fingerprint = None
    self.published = None
    self.address = None
    self.or_port = None
    self.dir_port = None

    self.flags = None

    self.version_line = None
    self.version = None

    self._unrecognized_lines = []

    entries = _get_descriptor_components(content, validate)

    if validate:
      self._check_constraints(entries)

    self._parse(entries, validate)
Example #17
0
  def __init__(self, raw_contents, validate = False):
    """
    Extra-info descriptor constructor. By default this validates the
    descriptor's content as it's parsed. This validation can be disabled to
    either improve performance or be accepting of malformed data.

    :param str raw_contents: extra-info content provided by the relay
    :param bool validate: checks the validity of the extra-info descriptor if
      **True**, skips these checks otherwise

    :raises: **ValueError** if the contents is malformed and validate is True
    """

    super(ExtraInfoDescriptor, self).__init__(raw_contents, lazy_load = not validate)
    entries = _get_descriptor_components(raw_contents, validate)

    if validate:
      for keyword in self._required_fields():
        if keyword not in entries:
          raise ValueError("Extra-info descriptor must have a '%s' entry" % keyword)

      for keyword in self._required_fields() + SINGLE_FIELDS:
        if keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("The '%s' entry can only appear once in an extra-info descriptor" % keyword)

      expected_first_keyword = self._first_keyword()
      if expected_first_keyword and expected_first_keyword != list(entries.keys())[0]:
        raise ValueError("Extra-info descriptor must start with a '%s' entry" % expected_first_keyword)

      expected_last_keyword = self._last_keyword()
      if expected_last_keyword and expected_last_keyword != list(entries.keys())[-1]:
        raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword)

      self._parse(entries, validate)
    else:
      self._entries = entries
Example #18
0
    def __init__(self, raw_contents, validate=True, annotations=None):
        """
    Server descriptor constructor, created from an individual relay's
    descriptor content (as provided by "GETINFO desc/*", cached descriptors,
    and metrics).

    By default this validates the descriptor's content as it's parsed. This
    validation can be disables to either improve performance or be accepting of
    malformed data.

    :param str raw_contents: descriptor content provided by the relay
    :param bool validate: checks the validity of the descriptor's content if
      **True**, skips these checks otherwise
    :param list annotations: lines that appeared prior to the descriptor

    :raises: **ValueError** if the contents is malformed and validate is True
    """

        super(ServerDescriptor, self).__init__(raw_contents)

        # Only a few things can be arbitrary bytes according to the dir-spec, so
        # parsing them separately.

        self.platform = _get_bytes_field("platform", raw_contents)
        self.contact = _get_bytes_field("contact", raw_contents)

        raw_contents = stem.util.str_tools._to_unicode(raw_contents)

        self.nickname = None
        self.fingerprint = None
        self.published = None

        self.address = None
        self.or_port = None
        self.socks_port = None
        self.dir_port = None

        self.tor_version = None
        self.operating_system = None
        self.uptime = None
        self.exit_policy = None
        self.exit_policy_v6 = DEFAULT_IPV6_EXIT_POLICY
        self.family = set()

        self.average_bandwidth = None
        self.burst_bandwidth = None
        self.observed_bandwidth = None

        self.link_protocols = None
        self.circuit_protocols = None
        self.hibernating = False
        self.allow_single_hop_exits = False
        self.extra_info_cache = False
        self.extra_info_digest = None
        self.hidden_service_dir = None
        self.eventdns = None
        self.or_addresses = []

        self.read_history_end = None
        self.read_history_interval = None
        self.read_history_values = None

        self.write_history_end = None
        self.write_history_interval = None
        self.write_history_values = None

        self._unrecognized_lines = []

        self._annotation_lines = annotations if annotations else []

        # A descriptor contains a series of 'keyword lines' which are simply a
        # keyword followed by an optional value. Lines can also be followed by a
        # signature block.
        #
        # We care about the ordering of 'accept' and 'reject' entries because this
        # influences the resulting exit policy, but for everything else the order
        # does not matter so breaking it into key / value pairs.

        entries, policy = _get_descriptor_components(raw_contents, validate,
                                                     ("accept", "reject"))

        if policy == [u'reject *:*']:
            self.exit_policy = REJECT_ALL_POLICY
        else:
            self.exit_policy = stem.exit_policy.ExitPolicy(*policy)

        self._parse(entries, validate)

        if validate:
            self._check_constraints(entries)
    def __init__(self, raw_contents, validate=True):
        """
    Extra-info descriptor constructor. By default this validates the
    descriptor's content as it's parsed. This validation can be disabled to
    either improve performance or be accepting of malformed data.

    :param str raw_contents: extra-info content provided by the relay
    :param bool validate: checks the validity of the extra-info descriptor if
      **True**, skips these checks otherwise

    :raises: **ValueError** if the contents is malformed and validate is True
    """

        super(ExtraInfoDescriptor, self).__init__(raw_contents)
        raw_contents = stem.util.str_tools._to_unicode(raw_contents)

        self.nickname = None
        self.fingerprint = None
        self.published = None
        self.geoip_db_digest = None
        self.geoip6_db_digest = None
        self.transport = {}

        self.conn_bi_direct_end = None
        self.conn_bi_direct_interval = None
        self.conn_bi_direct_below = None
        self.conn_bi_direct_read = None
        self.conn_bi_direct_write = None
        self.conn_bi_direct_both = None

        self.read_history_end = None
        self.read_history_interval = None
        self.read_history_values = None

        self.write_history_end = None
        self.write_history_interval = None
        self.write_history_values = None

        self.cell_stats_end = None
        self.cell_stats_interval = None
        self.cell_processed_cells = None
        self.cell_queued_cells = None
        self.cell_time_in_queue = None
        self.cell_circuits_per_decile = None

        self.dir_stats_end = None
        self.dir_stats_interval = None
        self.dir_v2_ips = None
        self.dir_v3_ips = None
        self.dir_v2_share = None
        self.dir_v3_share = None
        self.dir_v2_requests = None
        self.dir_v3_requests = None
        self.dir_v2_responses = None
        self.dir_v3_responses = None
        self.dir_v2_responses_unknown = None
        self.dir_v3_responses_unknown = None
        self.dir_v2_direct_dl = None
        self.dir_v3_direct_dl = None
        self.dir_v2_direct_dl_unknown = None
        self.dir_v3_direct_dl_unknown = None
        self.dir_v2_tunneled_dl = None
        self.dir_v3_tunneled_dl = None
        self.dir_v2_tunneled_dl_unknown = None
        self.dir_v3_tunneled_dl_unknown = None

        self.dir_read_history_end = None
        self.dir_read_history_interval = None
        self.dir_read_history_values = None

        self.dir_write_history_end = None
        self.dir_write_history_interval = None
        self.dir_write_history_values = None

        self.entry_stats_end = None
        self.entry_stats_interval = None
        self.entry_ips = None

        self.exit_stats_end = None
        self.exit_stats_interval = None
        self.exit_kibibytes_written = None
        self.exit_kibibytes_read = None
        self.exit_streams_opened = None

        self.bridge_stats_end = None
        self.bridge_stats_interval = None
        self.bridge_ips = None
        self.geoip_start_time = None
        self.geoip_client_origins = None

        self.ip_versions = None
        self.ip_transports = None

        self._unrecognized_lines = []

        entries = _get_descriptor_components(raw_contents, validate)

        if validate:
            for keyword in self._required_fields():
                if not keyword in entries:
                    raise ValueError(
                        "Extra-info descriptor must have a '%s' entry" %
                        keyword)

            for keyword in self._required_fields() + SINGLE_FIELDS:
                if keyword in entries and len(entries[keyword]) > 1:
                    raise ValueError(
                        "The '%s' entry can only appear once in an extra-info descriptor"
                        % keyword)

            expected_first_keyword = self._first_keyword()
            if expected_first_keyword and expected_first_keyword != entries.keys(
            )[0]:
                raise ValueError(
                    "Extra-info descriptor must start with a '%s' entry" %
                    expected_first_keyword)

            expected_last_keyword = self._last_keyword()
            if expected_last_keyword and expected_last_keyword != entries.keys(
            )[-1]:
                raise ValueError("Descriptor must end with a '%s' entry" %
                                 expected_last_keyword)

        self._parse(entries, validate)
  def __init__(self, raw_contents, validate = True, annotations = None):
    """
    Server descriptor constructor, created from an individual relay's
    descriptor content (as provided by 'GETINFO desc/*', cached descriptors,
    and metrics).

    By default this validates the descriptor's content as it's parsed. This
    validation can be disables to either improve performance or be accepting of
    malformed data.

    :param str raw_contents: descriptor content provided by the relay
    :param bool validate: checks the validity of the descriptor's content if
      **True**, skips these checks otherwise
    :param list annotations: lines that appeared prior to the descriptor

    :raises: **ValueError** if the contents is malformed and validate is True
    """

    super(ServerDescriptor, self).__init__(raw_contents)

    # Only a few things can be arbitrary bytes according to the dir-spec, so
    # parsing them separately.

    self.platform = _get_bytes_field('platform', raw_contents)
    self.contact = _get_bytes_field('contact', raw_contents)

    raw_contents = stem.util.str_tools._to_unicode(raw_contents)

    self.nickname = None
    self.fingerprint = None
    self.published = None

    self.address = None
    self.or_port = None
    self.socks_port = None
    self.dir_port = None

    self.tor_version = None
    self.operating_system = None
    self.uptime = None
    self.exit_policy = None
    self.exit_policy_v6 = DEFAULT_IPV6_EXIT_POLICY
    self.family = set()

    self.average_bandwidth = None
    self.burst_bandwidth = None
    self.observed_bandwidth = None

    self.link_protocols = None
    self.circuit_protocols = None
    self.hibernating = False
    self.allow_single_hop_exits = False
    self.extra_info_cache = False
    self.extra_info_digest = None
    self.hidden_service_dir = None
    self.eventdns = None
    self.or_addresses = []

    self.read_history_end = None
    self.read_history_interval = None
    self.read_history_values = None

    self.write_history_end = None
    self.write_history_interval = None
    self.write_history_values = None

    self._unrecognized_lines = []

    self._annotation_lines = annotations if annotations else []

    # A descriptor contains a series of 'keyword lines' which are simply a
    # keyword followed by an optional value. Lines can also be followed by a
    # signature block.
    #
    # We care about the ordering of 'accept' and 'reject' entries because this
    # influences the resulting exit policy, but for everything else the order
    # does not matter so breaking it into key / value pairs.

    entries, policy = _get_descriptor_components(raw_contents, validate, ('accept', 'reject'))

    if policy == [u'reject *:*']:
      self.exit_policy = REJECT_ALL_POLICY
    else:
      self.exit_policy = stem.exit_policy.ExitPolicy(*policy)

    self._parse(entries, validate)

    if validate:
      self._check_constraints(entries)
  def __init__(self, raw_contents, validate = True):
    """
    Extra-info descriptor constructor. By default this validates the
    descriptor's content as it's parsed. This validation can be disabled to
    either improve performance or be accepting of malformed data.

    :param str raw_contents: extra-info content provided by the relay
    :param bool validate: checks the validity of the extra-info descriptor if
      **True**, skips these checks otherwise

    :raises: **ValueError** if the contents is malformed and validate is True
    """

    super(ExtraInfoDescriptor, self).__init__(raw_contents)
    raw_contents = stem.util.str_tools._to_unicode(raw_contents)

    self.nickname = None
    self.fingerprint = None
    self.published = None
    self.geoip_db_digest = None
    self.geoip6_db_digest = None
    self.transport = {}

    self.conn_bi_direct_end = None
    self.conn_bi_direct_interval = None
    self.conn_bi_direct_below = None
    self.conn_bi_direct_read = None
    self.conn_bi_direct_write = None
    self.conn_bi_direct_both = None

    self.read_history_end = None
    self.read_history_interval = None
    self.read_history_values = None

    self.write_history_end = None
    self.write_history_interval = None
    self.write_history_values = None

    self.cell_stats_end = None
    self.cell_stats_interval = None
    self.cell_processed_cells = None
    self.cell_queued_cells = None
    self.cell_time_in_queue = None
    self.cell_circuits_per_decile = None

    self.dir_stats_end = None
    self.dir_stats_interval = None
    self.dir_v2_ips = None
    self.dir_v3_ips = None
    self.dir_v2_share = None
    self.dir_v3_share = None
    self.dir_v2_requests = None
    self.dir_v3_requests = None
    self.dir_v2_responses = None
    self.dir_v3_responses = None
    self.dir_v2_responses_unknown = None
    self.dir_v3_responses_unknown = None
    self.dir_v2_direct_dl = None
    self.dir_v3_direct_dl = None
    self.dir_v2_direct_dl_unknown = None
    self.dir_v3_direct_dl_unknown = None
    self.dir_v2_tunneled_dl = None
    self.dir_v3_tunneled_dl = None
    self.dir_v2_tunneled_dl_unknown = None
    self.dir_v3_tunneled_dl_unknown = None

    self.dir_read_history_end = None
    self.dir_read_history_interval = None
    self.dir_read_history_values = None

    self.dir_write_history_end = None
    self.dir_write_history_interval = None
    self.dir_write_history_values = None

    self.entry_stats_end = None
    self.entry_stats_interval = None
    self.entry_ips = None

    self.exit_stats_end = None
    self.exit_stats_interval = None
    self.exit_kibibytes_written = None
    self.exit_kibibytes_read = None
    self.exit_streams_opened = None

    self.bridge_stats_end = None
    self.bridge_stats_interval = None
    self.bridge_ips = None
    self.geoip_start_time = None
    self.geoip_client_origins = None

    self.ip_versions = None
    self.ip_transports = None

    self._unrecognized_lines = []

    entries = _get_descriptor_components(raw_contents, validate)

    if validate:
      for keyword in self._required_fields():
        if not keyword in entries:
          raise ValueError("Extra-info descriptor must have a '%s' entry" % keyword)

      for keyword in self._required_fields() + SINGLE_FIELDS:
        if keyword in entries and len(entries[keyword]) > 1:
          raise ValueError("The '%s' entry can only appear once in an extra-info descriptor" % keyword)

      expected_first_keyword = self._first_keyword()
      if expected_first_keyword and expected_first_keyword != entries.keys()[0]:
        raise ValueError("Extra-info descriptor must start with a '%s' entry" % expected_first_keyword)

      expected_last_keyword = self._last_keyword()
      if expected_last_keyword and expected_last_keyword != entries.keys()[-1]:
        raise ValueError("Descriptor must end with a '%s' entry" % expected_last_keyword)

    self._parse(entries, validate)