def _parse_microdescriptor_m_line(descriptor, entries):
# "m" digest
# example: m aiUklwBrua82obG5AsTX+iEpkjQA2+AQHxZ7GwMfY70
descriptor.microdescriptor_digest = _value('m', entries)
# TODO: drop the following in stem 2.x
descriptor.digest = _base64_to_hex(_value('m', entries), check_if_fingerprint = False)
def _parse_hidden_service_dir_line(descriptor, entries):
value = _value('hidden-service-dir', entries)
if value:
descriptor.hidden_service_dir = value.split(' ')
else:
descriptor.hidden_service_dir = ['2']
def _parse_id_line(descriptor, entries):
# "id" "ed25519" ed25519-identity
#
# examples:
#
# id ed25519 none
# id ed25519 8RH34kO07Pp+XYwzdoATVyCibIvmbslUjRkAm7J4IA8
value = _value('id', entries)
if value:
if not (descriptor.document and descriptor.document.is_vote):
vote_status = 'vote' if descriptor.document else '<undefined document>'
raise ValueError(
"%s 'id' line should only appear in votes (appeared in a %s): id %s"
% (descriptor._name(), vote_status, value))
value_comp = value.split()
if len(value_comp) >= 2:
descriptor.identifier_type = value_comp[0]
descriptor.identifier = value_comp[1]
else:
raise ValueError(
"'id' lines should contain both the key type and digest: id %s"
% value)
def _parse_dirreq_line(keyword, recognized_counts_attr,
unrecognized_counts_attr, descriptor, entries):
value = _value(keyword, entries)
recognized_counts = {}
unrecognized_counts = {}
is_response_stats = keyword in ('dirreq-v2-resp', 'dirreq-v3-resp')
key_set = DirResponse if is_response_stats else DirStat
key_type = 'STATUS' if is_response_stats else 'STAT'
for status, count in _mappings_for(keyword, value, divider=','):
if not count.isdigit():
raise ValueError(
'%s lines should contain %s=COUNT mappings: %s %s' %
(keyword, key_type, keyword, value))
if status in key_set:
recognized_counts[status] = int(count)
else:
unrecognized_counts[status] = int(count)
setattr(descriptor, recognized_counts_attr, recognized_counts)
setattr(descriptor, unrecognized_counts_attr, unrecognized_counts)
def _parse_w_line(descriptor, entries):
# "w" "Bandwidth=" INT ["Measured=" INT] ["Unmeasured=1"]
# example: w Bandwidth=7980
value = _value('w', entries)
w_comp = value.split(' ')
if len(w_comp) < 1:
raise ValueError("%s 'w' line is blank: w %s" % (descriptor._name(), value))
elif not w_comp[0].startswith('Bandwidth='):
raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (descriptor._name(), value))
for w_entry in w_comp:
if '=' in w_entry:
w_key, w_value = w_entry.split('=', 1)
else:
w_key, w_value = w_entry, None
if w_key == 'Bandwidth':
if not (w_value and w_value.isdigit()):
raise ValueError("%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (descriptor._name(), value))
descriptor.bandwidth = int(w_value)
elif w_key == 'Measured':
if not (w_value and w_value.isdigit()):
raise ValueError("%s 'Measured=' entry needs to have a numeric value: w %s" % (descriptor._name(), value))
descriptor.measured = int(w_value)
elif w_key == 'Unmeasured':
if w_value != '1':
raise ValueError("%s 'Unmeasured=' should only have the value of '1': w %s" % (descriptor._name(), value))
descriptor.is_unmeasured = True
else:
descriptor.unrecognized_bandwidth_entries.append(w_entry)
def _parse_hs_stats(keyword, stat_attribute, extra_attribute, descriptor,
entries):
# "<keyword>" num key=val key=val...
value, stat, extra = _value(keyword, entries), None, {}
if value is None:
pass # not in the descriptor
elif value == '':
raise ValueError("'%s' line was blank" % keyword)
else:
if ' ' in value:
stat_value, remainder = value.split(' ', 1)
else:
stat_value, remainder = value, None
try:
stat = int(stat_value)
except ValueError:
raise ValueError("'%s' stat was non-numeric (%s): %s %s" %
(keyword, stat_value, keyword, value))
for key, val in _mappings_for(keyword, remainder):
extra[key] = val
setattr(descriptor, stat_attribute, stat)
setattr(descriptor, extra_attribute, extra)
def _parse_id_line(descriptor: 'stem.descriptor.Descriptor',
entries: ENTRY_TYPE) -> None:
# "id" "ed25519" ed25519-identity
#
# examples:
#
# id ed25519 none
# id ed25519 8RH34kO07Pp+XYwzdoATVyCibIvmbslUjRkAm7J4IA8
value = _value('id', entries)
if value:
if descriptor.document and not descriptor.document.is_vote:
raise ValueError(
"%s 'id' line should only appear in votes: id %s" %
(descriptor._name(), value))
value_comp = value.split()
if len(value_comp) >= 2:
descriptor.identifier_type = value_comp[0]
descriptor.identifier = value_comp[1]
else:
raise ValueError(
"'id' lines should contain both the key type and digest: id %s"
% value)
def _parse_router_line(descriptor, entries):
# "router" nickname address ORPort SocksPort DirPort
value = _value('router', entries)
router_comp = value.split()
if len(router_comp) < 5:
raise ValueError('Router line must have five values: router %s' %
value)
elif not stem.util.tor_tools.is_valid_nickname(router_comp[0]):
raise ValueError("Router line entry isn't a valid nickname: %s" %
router_comp[0])
elif not stem.util.connection.is_valid_ipv4_address(router_comp[1]):
raise ValueError("Router line entry isn't a valid IPv4 address: %s" %
router_comp[1])
elif not stem.util.connection.is_valid_port(router_comp[2],
allow_zero=True):
raise ValueError("Router line's ORPort is invalid: %s" %
router_comp[2])
elif not stem.util.connection.is_valid_port(router_comp[3],
allow_zero=True):
raise ValueError("Router line's SocksPort is invalid: %s" %
router_comp[3])
elif not stem.util.connection.is_valid_port(router_comp[4],
allow_zero=True):
raise ValueError("Router line's DirPort is invalid: %s" %
router_comp[4])
descriptor.nickname = router_comp[0]
descriptor.address = router_comp[1]
descriptor.or_port = int(router_comp[2])
descriptor.socks_port = None if router_comp[3] == '0' else int(
router_comp[3])
descriptor.dir_port = None if router_comp[4] == '0' else int(
router_comp[4])
def _parse_version_line(descriptor, entries):
value = _value('version', entries)
if value.isdigit():
descriptor.version = int(value)
else:
raise ValueError('version line must have a positive integer value: %s' % value)
def _parse_hs_stats(keyword, stat_attribute, extra_attribute, descriptor, entries):
# "<keyword>" num key=val key=val...
value, stat, extra = _value(keyword, entries), None, {}
if value is not None:
value_comp = value.split()
if not value_comp:
raise ValueError("'%s' line was blank" % keyword)
try:
stat = int(value_comp[0])
except ValueError:
raise ValueError("'%s' stat was non-numeric (%s): %s %s" % (keyword, value_comp[0], keyword, value))
for entry in value_comp[1:]:
if '=' not in entry:
raise ValueError('Entries after the stat in %s lines should only be key=val entries: %s %s' % (keyword, keyword, value))
key, val = entry.split('=', 1)
extra[key] = val
setattr(descriptor, stat_attribute, stat)
setattr(descriptor, extra_attribute, extra)
def _parse_hidden_service_dir_line(descriptor, entries):
value = _value('hidden-service-dir', entries)
if value:
descriptor.hidden_service_dir = value.split(' ')
else:
descriptor.hidden_service_dir = ['2']
def _parse_padding_counts_line(descriptor, entries):
# "padding-counts" YYYY-MM-DD HH:MM:SS (NSEC s) key=val key=val...
value = _value('padding-counts', entries)
timestamp, interval, remainder = _parse_timestamp_and_interval(
'padding-counts', value)
entries = {}
for entry in remainder.split(' '):
if '=' not in entry:
raise ValueError(
'Entries in padding-counts line should be key=value mappings: padding-counts %s'
% value)
k, v = entry.split('=', 1)
if not v:
raise ValueError(
'Entry in padding-counts line had a blank value: padding-counts %s'
% value)
entries[k] = int(v) if v.isdigit() else v
setattr(descriptor, 'padding_counts_end', timestamp)
setattr(descriptor, 'padding_counts_interval', interval)
setattr(descriptor, 'padding_counts', entries)
def _parse_version_line(descriptor, entries):
value = _value('version', entries)
if value.isdigit():
descriptor.version = int(value)
else:
raise ValueError('version line must have a positive integer value: %s' % value)
def _parse_dirreq_line(keyword, recognized_counts_attr,
unrecognized_counts_attr, descriptor, entries):
value = _value(keyword, entries)
recognized_counts = {}
unrecognized_counts = {}
is_response_stats = keyword in ('dirreq-v2-resp', 'dirreq-v3-resp')
key_set = DirResponse if is_response_stats else DirStat
key_type = 'STATUS' if is_response_stats else 'STAT'
error_msg = '%s lines should contain %s=COUNT mappings: %s %s' % (
keyword, key_type, keyword, value)
if value:
for entry in value.split(','):
if '=' not in entry:
raise ValueError(error_msg)
status, count = entry.split('=', 1)
if count.isdigit():
if status in key_set:
recognized_counts[status] = int(count)
else:
unrecognized_counts[status] = int(count)
else:
raise ValueError(error_msg)
setattr(descriptor, recognized_counts_attr, recognized_counts)
setattr(descriptor, unrecognized_counts_attr, unrecognized_counts)
def _parse_dirreq_line(keyword, recognized_counts_attr, unrecognized_counts_attr, descriptor, entries):
value = _value(keyword, entries)
recognized_counts = {}
unrecognized_counts = {}
is_response_stats = keyword in ('dirreq-v2-resp', 'dirreq-v3-resp')
key_set = DirResponse if is_response_stats else DirStat
key_type = 'STATUS' if is_response_stats else 'STAT'
error_msg = '%s lines should contain %s=COUNT mappings: %s %s' % (keyword, key_type, keyword, value)
if value:
for entry in value.split(','):
if '=' not in entry:
raise ValueError(error_msg)
status, count = entry.split('=', 1)
if count.isdigit():
if status in key_set:
recognized_counts[status] = int(count)
else:
unrecognized_counts[status] = int(count)
else:
raise ValueError(error_msg)
setattr(descriptor, recognized_counts_attr, recognized_counts)
setattr(descriptor, unrecognized_counts_attr, unrecognized_counts)
def _parse_hs_stats(keyword, stat_attribute, extra_attribute, descriptor,
entries):
# "<keyword>" num key=val key=val...
value, stat, extra = _value(keyword, entries), None, {}
if value is not None:
value_comp = value.split()
if not value_comp:
raise ValueError("'%s' line was blank" % keyword)
try:
stat = int(value_comp[0])
except ValueError:
raise ValueError("'%s' stat was non-numeric (%s): %s %s" %
(keyword, value_comp[0], keyword, value))
for entry in value_comp[1:]:
if '=' not in entry:
raise ValueError(
'Entries after the stat in %s lines should only be key=val entries: %s %s'
% (keyword, keyword, value))
key, val = entry.split('=', 1)
extra[key] = val
setattr(descriptor, stat_attribute, stat)
setattr(descriptor, extra_attribute, extra)
def _parse_geoip_to_count_line(keyword, attribute, descriptor, entries):
# "<keyword>" CC=N,CC=N,...
#
# The maxmind geoip (https://www.maxmind.com/app/iso3166) has numeric
# locale codes for some special values, for instance...
# A1,"Anonymous Proxy"
# A2,"Satellite Provider"
# ??,"Unknown"
value, locale_usage = _value(keyword, entries), {}
error_msg = 'Entries in %s line should only be CC=N entries: %s %s' % (
keyword, keyword, value)
if value:
for entry in value.split(','):
if '=' not in entry:
raise ValueError(error_msg)
locale, count = entry.split('=', 1)
if _locale_re.match(locale) and count.isdigit():
locale_usage[locale] = int(count)
else:
raise ValueError(error_msg)
setattr(descriptor, attribute, locale_usage)
def _parse_geoip_to_count_line(keyword, attribute, descriptor, entries):
# "<keyword>" CC=N,CC=N,...
#
# The maxmind geoip (https://www.maxmind.com/app/iso3166) has numeric
# locale codes for some special values, for instance...
# A1,"Anonymous Proxy"
# A2,"Satellite Provider"
# ??,"Unknown"
value, locale_usage = _value(keyword, entries), {}
error_msg = 'Entries in %s line should only be CC=N entries: %s %s' % (keyword, keyword, value)
if value:
for entry in value.split(','):
if '=' not in entry:
raise ValueError(error_msg)
locale, count = entry.split('=', 1)
if _locale_re.match(locale) and count.isdigit():
locale_usage[locale] = int(count)
else:
raise ValueError(error_msg)
setattr(descriptor, attribute, locale_usage)
def _parse_platform_line(descriptor: 'stem.descriptor.Descriptor',
entries: ENTRY_TYPE) -> None:
# "platform" string
_parse_bytes_line('platform', 'platform')(descriptor, entries)
# The platform attribute was set earlier. This line can contain any
# arbitrary data, but tor seems to report its version followed by the
# os like the following...
#
# platform Tor 0.2.2.35 (git-73ff13ab3cc9570d) on Linux x86_64
#
# There's no guarantee that we'll be able to pick these out the
# version, but might as well try to save our caller the effort.
value = _value('platform', entries)
platform_match = re.match('^(?:node-)?Tor (\\S*).* on (.*)$', value)
if platform_match:
version_str, descriptor.operating_system = platform_match.groups()
try:
descriptor.tor_version = stem.version._get_version(version_str)
except ValueError:
pass
def _parse_extrainfo_digest_line(descriptor, entries):
value = _value('extra-info-digest', entries)
value = value.split(' ')[0] # lines have additional content from propsal 228, waiting for it to be documented: #16227
if not stem.util.tor_tools.is_hex_digits(value, 40):
raise ValueError('extra-info-digest should be 40 hex characters: %s' % value)
descriptor.extra_info_digest = value
def _parse_create2_formats_line(descriptor, entries):
value = _value('create2-formats', entries)
if value.isdigit():
descriptor.create2_formats = int(value)
else:
raise ValueError(
'create2-formats line must have a positive integer value: %s' %
value)
def _parse_hibernating_line(descriptor, entries):
# "hibernating" 0|1 (in practice only set if one)
value = _value('hibernating', entries)
if value not in ('0', '1'):
raise ValueError('Hibernating line had an invalid value, must be zero or one: %s' % value)
descriptor.hibernating = value == '1'
def _parse_extrainfo_digest_line(descriptor, entries):
value = _value('extra-info-digest', entries)
digest_comp = value.split(' ')
if not stem.util.tor_tools.is_hex_digits(digest_comp[0], 40):
raise ValueError('extra-info-digest should be 40 hex characters: %s' % digest_comp[0])
descriptor.extra_info_digest = digest_comp[0]
descriptor.extra_info_sha256_digest = digest_comp[1] if len(digest_comp) >= 2 else None
def _parse_revision_counter(descriptor, entries):
value = _value('revision-counter', entries)
if value.isdigit():
descriptor.revision_counter = int(value)
else:
raise ValueError(
'revision-counter line must have a positive integer value: %s' %
value)
def _parse_hs_descriptor_lifetime(descriptor, entries):
value = _value('descriptor-lifetime', entries)
if value.isdigit():
descriptor.descriptor_lifetime = int(value)
else:
raise ValueError(
'descriptor-lifetime line must have a positive integer value: %s' %
value)
def _parse_extrainfo_digest_line(descriptor, entries):
value = _value('extra-info-digest', entries)
digest_comp = value.split(' ')
if not stem.util.tor_tools.is_hex_digits(digest_comp[0], 40):
raise ValueError('extra-info-digest should be 40 hex characters: %s' % digest_comp[0])
descriptor.extra_info_digest = digest_comp[0]
descriptor.extra_info_sha256_digest = digest_comp[1] if len(digest_comp) >= 2 else None
def _parse_id_line(descriptor, entries):
value = _value('id', entries)
value_comp = value.split()
if len(value_comp) >= 2:
descriptor.identifier_type = value_comp[0]
descriptor.identifier = value_comp[1]
else:
raise ValueError("'id' lines should contain both the key type and digest: id %s" % value)
def _parse_timestamp_and_interval_line(keyword, end_attribute,
interval_attribute, descriptor,
entries):
# "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s)
timestamp, interval, _ = _parse_timestamp_and_interval(
keyword, _value(keyword, entries))
setattr(descriptor, end_attribute, timestamp)
setattr(descriptor, interval_attribute, interval)
def _parse_hibernating_line(descriptor, entries):
# "hibernating" 0|1 (in practice only set if one)
value = _value('hibernating', entries)
if value not in ('0', '1'):
raise ValueError('Hibernating line had an invalid value, must be zero or one: %s' % value)
descriptor.hibernating = value == '1'
def _parse_r_line(descriptor: 'stem.descriptor.Descriptor',
entries: ENTRY_TYPE) -> None:
# Parses a RouterStatusEntry's 'r' line. They're very nearly identical for
# all current entry types (v2, v3, and microdescriptor v3) with one little
# wrinkle: only the microdescriptor flavor excludes a 'digest' field.
#
# For v2 and v3 router status entries:
# "r" nickname identity digest publication IP ORPort DirPort
# example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0
#
# For v3 microdescriptor router status entries:
# "r" nickname identity publication IP ORPort DirPort
# example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030
value = _value('r', entries)
include_digest = not isinstance(descriptor, RouterStatusEntryMicroV3)
r_comp = value.split(' ')
# inject a None for the digest to normalize the field positioning
if not include_digest:
r_comp.insert(2, None)
if len(r_comp) < 8:
expected_field_count = 'eight' if include_digest else 'seven'
raise ValueError("%s 'r' line must have %s values: r %s" %
(descriptor._name(), expected_field_count, value))
if not stem.util.tor_tools.is_valid_nickname(r_comp[0]):
raise ValueError("%s nickname isn't valid: %s" %
(descriptor._name(), r_comp[0]))
elif not stem.util.connection.is_valid_ipv4_address(r_comp[5]):
raise ValueError("%s address isn't a valid IPv4 address: %s" %
(descriptor._name(), r_comp[5]))
elif not stem.util.connection.is_valid_port(r_comp[6]):
raise ValueError('%s ORPort is invalid: %s' %
(descriptor._name(), r_comp[6]))
elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero=True):
raise ValueError('%s DirPort is invalid: %s' %
(descriptor._name(), r_comp[7]))
descriptor.nickname = r_comp[0]
descriptor.fingerprint = _base64_to_hex(r_comp[1])
if include_digest:
descriptor.digest = _base64_to_hex(r_comp[2])
descriptor.address = r_comp[5]
descriptor.or_port = int(r_comp[6])
descriptor.dir_port = None if r_comp[7] == '0' else int(r_comp[7])
try:
published = '%s %s' % (r_comp[3], r_comp[4])
descriptor.published = stem.util.str_tools._parse_timestamp(published)
except ValueError:
raise ValueError("Publication time time wasn't parsable: r %s" % value)
def _parse_protocols_line(descriptor, entries):
value = _value('protocols', entries)
protocols_match = re.match('^Link (.*) Circuit (.*)$', value)
if not protocols_match:
raise ValueError('Protocols line did not match the expected pattern: protocols %s' % value)
link_versions, circuit_versions = protocols_match.groups()
descriptor.link_protocols = link_versions.split(' ')
descriptor.circuit_protocols = circuit_versions.split(' ')
def _parse_bridge_ip_versions_line(descriptor: 'stem.descriptor.Descriptor', entries: ENTRY_TYPE) -> None:
value, ip_versions = _value('bridge-ip-versions', entries), {}
for protocol, count in _mappings_for('bridge-ip-versions', value, divider = ','):
if not count.isdigit():
raise stem.ProtocolError('IP protocol count was non-numeric (%s): bridge-ip-versions %s' % (count, value))
ip_versions[protocol] = int(count)
descriptor.ip_versions = ip_versions
def _parse_bridge_ip_transports_line(descriptor, entries):
value, ip_transports = _value('bridge-ip-transports', entries), {}
for protocol, count in _mappings_for('bridge-ip-transports', value, divider = ','):
if not count.isdigit():
raise stem.ProtocolError('Transport count was non-numeric (%s): bridge-ip-transports %s' % (count, value))
ip_transports[protocol] = int(count)
descriptor.ip_transports = ip_transports
def _parse_protocols_line(descriptor, entries):
value = _value('protocols', entries)
protocols_match = re.match('^Link (.*) Circuit (.*)$', value)
if not protocols_match:
raise ValueError('Protocols line did not match the expected pattern: protocols %s' % value)
link_versions, circuit_versions = protocols_match.groups()
descriptor.link_protocols = link_versions.split(' ')
descriptor.circuit_protocols = circuit_versions.split(' ')
def _parse_w_line(descriptor: 'stem.descriptor.Descriptor',
entries: ENTRY_TYPE) -> None:
# "w" "Bandwidth=" INT ["Measured=" INT] ["Unmeasured=1"]
# example: w Bandwidth=7980
value = _value('w', entries)
w_comp = value.split(' ')
if len(w_comp) < 1:
raise ValueError("%s 'w' line is blank: w %s" %
(descriptor._name(), value))
elif not w_comp[0].startswith('Bandwidth='):
raise ValueError(
"%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" %
(descriptor._name(), value))
bandwidth = None
measured = None
is_unmeasured = False
unrecognized_bandwidth_entries = []
for w_entry in w_comp:
if '=' in w_entry:
w_key, w_value = w_entry.split('=', 1)
else:
w_key, w_value = w_entry, None
if w_key == 'Bandwidth':
if not (w_value and w_value.isdigit()):
raise ValueError(
"%s 'Bandwidth=' entry needs to have a numeric value: w %s"
% (descriptor._name(), value))
bandwidth = int(w_value)
elif w_key == 'Measured':
if not (w_value and w_value.isdigit()):
raise ValueError(
"%s 'Measured=' entry needs to have a numeric value: w %s"
% (descriptor._name(), value))
measured = int(w_value)
elif w_key == 'Unmeasured':
if w_value != '1':
raise ValueError(
"%s 'Unmeasured=' should only have the value of '1': w %s"
% (descriptor._name(), value))
is_unmeasured = True
else:
unrecognized_bandwidth_entries.append(w_entry)
descriptor.bandwidth = bandwidth
descriptor.measured = measured
descriptor.is_unmeasured = is_unmeasured
descriptor.unrecognized_bandwidth_entries = unrecognized_bandwidth_entries
def _parse_cell_circuits_per_decline_line(descriptor, entries):
# "cell-circuits-per-decile" num
value = _value('cell-circuits-per-decile', entries)
if not value.isdigit():
raise ValueError('Non-numeric cell-circuits-per-decile value: %s' % value)
elif int(value) < 0:
raise ValueError('Negative cell-circuits-per-decile value: %s' % value)
descriptor.cell_circuits_per_decile = int(value)
def _parse_dirreq_share_line(keyword, attribute, descriptor, entries):
value = _value(keyword, entries)
if not value.endswith('%'):
raise ValueError('%s lines should be a percentage: %s %s' % (keyword, keyword, value))
elif float(value[:-1]) < 0:
raise ValueError('Negative percentage value: %s %s' % (keyword, value))
# bug means it might be above 100%: https://lists.torproject.org/pipermail/tor-dev/2012-June/003679.html
setattr(descriptor, attribute, float(value[:-1]) / 100)
def _parse_id_line(descriptor, entries):
value = _value('id', entries)
value_comp = value.split()
if len(value_comp) >= 2:
descriptor.identifier_type = value_comp[0]
descriptor.identifier = value_comp[1]
else:
raise ValueError(
"'id' lines should contain both the key type and digest: id %s" %
value)
def _parse_p_line(descriptor, entries):
# "p" ("accept" / "reject") PortList
# p reject 1-65535
# example: p accept 80,110,143,443,993,995,6660-6669,6697,7000-7001
value = _value('p', entries)
try:
descriptor.exit_policy = stem.exit_policy.MicroExitPolicy(value)
except ValueError as exc:
raise ValueError('%s exit policy is malformed (%s): p %s' % (descriptor._name(), exc, value))
def _parse_dirreq_share_line(keyword, attribute, descriptor, entries):
value = _value(keyword, entries)
if not value.endswith('%'):
raise ValueError('%s lines should be a percentage: %s %s' % (keyword, keyword, value))
elif float(value[:-1]) < 0:
raise ValueError('Negative percentage value: %s %s' % (keyword, value))
# bug means it might be above 100%: https://lists.torproject.org/pipermail/tor-dev/2012-June/003679.html
setattr(descriptor, attribute, float(value[:-1]) / 100)
def _parse_cell_circuits_per_decline_line(descriptor, entries):
# "cell-circuits-per-decile" num
value = _value('cell-circuits-per-decile', entries)
if not value.isdigit():
raise ValueError('Non-numeric cell-circuits-per-decile value: %s' % value)
elif int(value) < 0:
raise ValueError('Negative cell-circuits-per-decile value: %s' % value)
descriptor.cell_circuits_per_decile = int(value)
def _parse_extrainfo_digest_line(descriptor, entries):
value = _value('extra-info-digest', entries)
value = value.split(
' '
)[0] # lines have additional content from propsal 228, waiting for it to be documented: #16227
if not stem.util.tor_tools.is_hex_digits(value, 40):
raise ValueError('extra-info-digest should be 40 hex characters: %s' %
value)
descriptor.extra_info_digest = value
def _parse_protocol_versions_line(descriptor, entries):
value = _value('protocol-versions', entries)
try:
versions = [int(entry) for entry in value.split(',')]
except ValueError:
raise ValueError('protocol-versions line has non-numeric versoins: protocol-versions %s' % value)
for v in versions:
if v <= 0:
raise ValueError('protocol-versions must be positive integers: %s' % value)
descriptor.protocol_versions = versions
def _parse_padding_counts_line(descriptor, entries):
# "padding-counts" YYYY-MM-DD HH:MM:SS (NSEC s) key=val key=val...
value = _value('padding-counts', entries)
timestamp, interval, remainder = _parse_timestamp_and_interval('padding-counts', value)
counts = {}
for k, v in _mappings_for('padding-counts', remainder, require_value = True):
counts[k] = int(v) if v.isdigit() else v
setattr(descriptor, 'padding_counts_end', timestamp)
setattr(descriptor, 'padding_counts_interval', interval)
setattr(descriptor, 'padding_counts', counts)
def _parse_s_line(descriptor, entries):
# "s" Flags
# example: s Named Running Stable Valid
value = _value('s', entries)
flags = [] if value == '' else value.split(' ')
descriptor.flags = flags
for flag in flags:
if flags.count(flag) > 1:
raise ValueError('%s had duplicate flags: s %s' % (descriptor._name(), value))
elif flag == '':
raise ValueError("%s had extra whitespace on its 's' line: s %s" % (descriptor._name(), value))
def _parse_port_count_line(keyword, attribute, descriptor, entries):
# "<keyword>" port=N,port=N,...
value, port_mappings = _value(keyword, entries), {}
for port, stat in _mappings_for(keyword, value, divider = ','):
if (port != 'other' and not stem.util.connection.is_valid_port(port)) or not stat.isdigit():
raise ValueError('Entries in %s line should only be PORT=N entries: %s %s' % (keyword, keyword, value))
port = int(port) if port.isdigit() else port
port_mappings[port] = int(stat)
setattr(descriptor, attribute, port_mappings)
def _parse_w_line(descriptor, entries):
# "w" "Bandwidth=" INT ["Measured=" INT] ["Unmeasured=1"]
# example: w Bandwidth=7980
value = _value("w", entries)
w_comp = value.split(" ")
if len(w_comp) < 1:
raise ValueError("%s 'w' line is blank: w %s" % (descriptor._name(), value))
elif not w_comp[0].startswith("Bandwidth="):
raise ValueError("%s 'w' line needs to start with a 'Bandwidth=' entry: w %s" % (descriptor._name(), value))
bandwidth = None
measured = None
is_unmeasured = False
unrecognized_bandwidth_entries = []
for w_entry in w_comp:
if "=" in w_entry:
w_key, w_value = w_entry.split("=", 1)
else:
w_key, w_value = w_entry, None
if w_key == "Bandwidth":
if not (w_value and w_value.isdigit()):
raise ValueError(
"%s 'Bandwidth=' entry needs to have a numeric value: w %s" % (descriptor._name(), value)
)
bandwidth = int(w_value)
elif w_key == "Measured":
if not (w_value and w_value.isdigit()):
raise ValueError(
"%s 'Measured=' entry needs to have a numeric value: w %s" % (descriptor._name(), value)
)
measured = int(w_value)
elif w_key == "Unmeasured":
if w_value != "1":
raise ValueError(
"%s 'Unmeasured=' should only have the value of '1': w %s" % (descriptor._name(), value)
)
is_unmeasured = True
else:
unrecognized_bandwidth_entries.append(w_entry)
descriptor.bandwidth = bandwidth
descriptor.measured = measured
descriptor.is_unmeasured = is_unmeasured
descriptor.unrecognized_bandwidth_entries = unrecognized_bandwidth_entries
def _parse_r_line(descriptor, entries):
# Parses a RouterStatusEntry's 'r' line. They're very nearly identical for
# all current entry types (v2, v3, and microdescriptor v3) with one little
# wrinkle: only the microdescriptor flavor excludes a 'digest' field.
#
# For v2 and v3 router status entries:
# "r" nickname identity digest publication IP ORPort DirPort
# example: r mauer BD7xbfsCFku3+tgybEZsg8Yjhvw itcuKQ6PuPLJ7m/Oi928WjO2j8g 2012-06-22 13:19:32 80.101.105.103 9001 0
#
# For v3 microdescriptor router status entries:
# "r" nickname identity publication IP ORPort DirPort
# example: r Konata ARIJF2zbqirB9IwsW0mQznccWww 2012-09-24 13:40:40 69.64.48.168 9001 9030
value = _value('r', entries)
include_digest = not isinstance(descriptor, RouterStatusEntryMicroV3)
r_comp = value.split(' ')
# inject a None for the digest to normalize the field positioning
if not include_digest:
r_comp.insert(2, None)
if len(r_comp) < 8:
expected_field_count = 'eight' if include_digest else 'seven'
raise ValueError("%s 'r' line must have %s values: r %s" % (descriptor._name(), expected_field_count, value))
if not stem.util.tor_tools.is_valid_nickname(r_comp[0]):
raise ValueError("%s nickname isn't valid: %s" % (descriptor._name(), r_comp[0]))
elif not stem.util.connection.is_valid_ipv4_address(r_comp[5]):
raise ValueError("%s address isn't a valid IPv4 address: %s" % (descriptor._name(), r_comp[5]))
elif not stem.util.connection.is_valid_port(r_comp[6]):
raise ValueError('%s ORPort is invalid: %s' % (descriptor._name(), r_comp[6]))
elif not stem.util.connection.is_valid_port(r_comp[7], allow_zero = True):
raise ValueError('%s DirPort is invalid: %s' % (descriptor._name(), r_comp[7]))
descriptor.nickname = r_comp[0]
descriptor.fingerprint = _base64_to_hex(r_comp[1])
if include_digest:
descriptor.digest = _base64_to_hex(r_comp[2])
descriptor.address = r_comp[5]
descriptor.or_port = int(r_comp[6])
descriptor.dir_port = None if r_comp[7] == '0' else int(r_comp[7])
try:
published = '%s %s' % (r_comp[3], r_comp[4])
descriptor.published = stem.util.str_tools._parse_timestamp(published)
except ValueError:
raise ValueError("Publication time time wasn't parsable: r %s" % value)
def _parse_history_line(keyword, history_end_attribute, history_interval_attribute, history_values_attribute, descriptor, entries):
value = _value(keyword, entries)
timestamp, interval, remainder = stem.descriptor.extrainfo_descriptor._parse_timestamp_and_interval(keyword, value)
try:
if remainder:
history_values = [int(entry) for entry in remainder.split(',')]
else:
history_values = []
except ValueError:
raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value))
setattr(descriptor, history_end_attribute, timestamp)
setattr(descriptor, history_interval_attribute, interval)
setattr(descriptor, history_values_attribute, history_values)
def _parse_extra_info_line(descriptor, entries):
# "extra-info" Nickname Fingerprint
value = _value('extra-info', entries)
extra_info_comp = value.split()
if len(extra_info_comp) < 2:
raise ValueError('Extra-info line must have two values: extra-info %s' % value)
elif not stem.util.tor_tools.is_valid_nickname(extra_info_comp[0]):
raise ValueError("Extra-info line entry isn't a valid nickname: %s" % extra_info_comp[0])
elif not stem.util.tor_tools.is_valid_fingerprint(extra_info_comp[1]):
raise ValueError('Tor relay fingerprints consist of forty hex digits: %s' % extra_info_comp[1])
descriptor.nickname = extra_info_comp[0]
descriptor.fingerprint = extra_info_comp[1]
def _parse_fingerprint_line(descriptor, entries):
# This is forty hex digits split into space separated groups of four.
# Checking that we match this pattern.
value = _value('fingerprint', entries)
fingerprint = value.replace(' ', '')
for grouping in value.split(' '):
if len(grouping) != 4:
raise ValueError('Fingerprint line should have groupings of four hex digits: %s' % value)
if not stem.util.tor_tools.is_valid_fingerprint(fingerprint):
raise ValueError('Tor relay fingerprints consist of forty hex digits: %s' % value)
descriptor.fingerprint = fingerprint
def _parse_conn_bi_direct_line(descriptor, entries):
# "conn-bi-direct" YYYY-MM-DD HH:MM:SS (NSEC s) BELOW,READ,WRITE,BOTH
value = _value('conn-bi-direct', entries)
timestamp, interval, remainder = _parse_timestamp_and_interval('conn-bi-direct', value)
stats = remainder.split(',')
if len(stats) != 4 or not (stats[0].isdigit() and stats[1].isdigit() and stats[2].isdigit() and stats[3].isdigit()):
raise ValueError('conn-bi-direct line should end with four numeric values: conn-bi-direct %s' % value)
descriptor.conn_bi_direct_end = timestamp
descriptor.conn_bi_direct_interval = interval
descriptor.conn_bi_direct_below = int(stats[0])
descriptor.conn_bi_direct_read = int(stats[1])
descriptor.conn_bi_direct_write = int(stats[2])
descriptor.conn_bi_direct_both = int(stats[3])
def _parse_history_line(keyword, end_attribute, interval_attribute, values_attribute, descriptor, entries):
# "<keyword>" YYYY-MM-DD HH:MM:SS (NSEC s) NUM,NUM,NUM,NUM,NUM...
value = _value(keyword, entries)
timestamp, interval, remainder = _parse_timestamp_and_interval(keyword, value)
history_values = []
if remainder:
try:
history_values = [int(entry) for entry in remainder.split(',')]
except ValueError:
raise ValueError('%s line has non-numeric values: %s %s' % (keyword, keyword, value))
setattr(descriptor, end_attribute, timestamp)
setattr(descriptor, interval_attribute, interval)
setattr(descriptor, values_attribute, history_values)
def _parse_bridge_ip_transports_line(descriptor, entries):
value, ip_transports = _value('bridge-ip-transports', entries), {}
if value:
for entry in value.split(','):
if '=' not in entry:
raise stem.ProtocolError("The bridge-ip-transports should be a comma separated listing of '<protocol>=<count>' mappings: bridge-ip-transports %s" % value)
protocol, count = entry.split('=', 1)
if not count.isdigit():
raise stem.ProtocolError('Transport count was non-numeric (%s): bridge-ip-transports %s' % (count, value))
ip_transports[protocol] = int(count)
descriptor.ip_transports = ip_transports
def _parse_v_line(descriptor, entries):
# "v" version
# example: v Tor 0.2.2.35
#
# The spec says that if this starts with "Tor " then what follows is a
# tor version. If not then it has "upgraded to a more sophisticated
# protocol versioning system".
value = _value('v', entries)
descriptor.version_line = value
if value.startswith('Tor '):
try:
descriptor.version = stem.version._get_version(value[4:])
except ValueError as exc:
raise ValueError('%s has a malformed tor version (%s): v %s' % (descriptor._name(), exc, value))
def _parse_bandwidth_line(descriptor, entries):
# "bandwidth" bandwidth-avg bandwidth-burst bandwidth-observed
value = _value('bandwidth', entries)
bandwidth_comp = value.split()
if len(bandwidth_comp) < 3:
raise ValueError('Bandwidth line must have three values: bandwidth %s' % value)
elif not bandwidth_comp[0].isdigit():
raise ValueError("Bandwidth line's average rate isn't numeric: %s" % bandwidth_comp[0])
elif not bandwidth_comp[1].isdigit():
raise ValueError("Bandwidth line's burst rate isn't numeric: %s" % bandwidth_comp[1])
elif not bandwidth_comp[2].isdigit():
raise ValueError("Bandwidth line's observed rate isn't numeric: %s" % bandwidth_comp[2])
descriptor.average_bandwidth = int(bandwidth_comp[0])
descriptor.burst_bandwidth = int(bandwidth_comp[1])
descriptor.observed_bandwidth = int(bandwidth_comp[2])
def _parse_uptime_line(descriptor, entries):
# We need to be tolerant of negative uptimes to accommodate a past tor
# bug...
#
# Changes in version 0.1.2.7-alpha - 2007-02-06
# - If our system clock jumps back in time, don't publish a negative
# uptime in the descriptor. Also, don't let the global rate limiting
# buckets go absurdly negative.
#
# After parsing all of the attributes we'll double check that negative
# uptimes only occurred prior to this fix.
value = _value('uptime', entries)
try:
descriptor.uptime = int(value)
except ValueError:
raise ValueError('Uptime line must have an integer value: %s' % value)
def _parse_cell_line(keyword, attribute, descriptor, entries):
# "<keyword>" num,...,num
value = _value(keyword, entries)
entries, exc = [], None
if value:
for entry in value.split(','):
try:
# Values should be positive but as discussed in ticket #5849
# there was a bug around this. It was fixed in tor 0.2.2.1.
entries.append(float(entry))
except ValueError:
exc = ValueError('Non-numeric entry in %s listing: %s %s' % (keyword, keyword, value))
setattr(descriptor, attribute, entries)
if exc:
raise exc
def _parse_id_line(descriptor, entries):
# "id" "ed25519" ed25519-identity
#
# examples:
#
# id ed25519 none
# id ed25519 8RH34kO07Pp+XYwzdoATVyCibIvmbslUjRkAm7J4IA8
value = _value('id', entries)
if value:
if descriptor.document and not descriptor.document.is_vote:
raise ValueError("%s 'id' line should only appear in votes: id %s" % (descriptor._name(), value))
value_comp = value.split()
if len(value_comp) >= 2:
descriptor.identifier_type = value_comp[0]
descriptor.identifier = value_comp[1]
else:
raise ValueError("'id' lines should contain both the key type and digest: id %s" % value)
def _parse_padding_counts_line(descriptor, entries):
# "padding-counts" YYYY-MM-DD HH:MM:SS (NSEC s) key=val key=val...
value = _value('padding-counts', entries)
timestamp, interval, remainder = _parse_timestamp_and_interval('padding-counts', value)
entries = {}
for entry in remainder.split(' '):
if '=' not in entry:
raise ValueError('Entries in padding-counts line should be key=value mappings: padding-counts %s' % value)
k, v = entry.split('=', 1)
if not v:
raise ValueError('Entry in padding-counts line had a blank value: padding-counts %s' % value)
entries[k] = int(v) if v.isdigit() else v
setattr(descriptor, 'padding_counts_end', timestamp)
setattr(descriptor, 'padding_counts_interval', interval)
setattr(descriptor, 'padding_counts', entries)
