def test_version_lists(self): """ Parses client-versions and server-versions fields. Both are comma separated lists of tor versions. """ expected = [stem.version.Version("1.2.3.4"), stem.version.Version("56.789.12.34-alpha")] test_value = "1.2.3.4,56.789.12.34-alpha" document = get_network_status_document_v3({"client-versions": test_value, "server-versions": test_value}) self.assertEquals(expected, document.client_versions) self.assertEquals(expected, document.server_versions) test_values = ( ("", []), (" ", []), ("1.2.3.4,", [stem.version.Version("1.2.3.4")]), ("1.2.3.4,1.2.3.a", [stem.version.Version("1.2.3.4")]), ) for field in ('client-versions', 'server-versions'): attr = field.replace('-', '_') for test_value, expected_value in test_values: content = get_network_status_document_v3({field: test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected_value, getattr(document, attr))
def test_examples(self): """ Run something similar to the examples in the header pydocs. """ # makes a consensus with a couple routers, both with the same nickname entry1 = get_router_status_entry_v3({'s': "Fast"}) entry2 = get_router_status_entry_v3({'s': "Valid"}) content = get_network_status_document_v3(routers=(entry1, entry2), content=True) # first example: parsing via the NetworkStatusDocumentV3 constructor consensus_file = StringIO.StringIO(content) consensus = NetworkStatusDocumentV3(consensus_file.read()) consensus_file.close() for router in consensus.routers: self.assertEqual('caerSidi', router.nickname) # second example: using parse_file with support_with(StringIO.StringIO(content)) as consensus_file: for router in parse_file(consensus_file): self.assertEqual('caerSidi', router.nickname)
def test_footer_consensus_method_requirement(self): """ Check that validation will notice if a footer appears before it was introduced. """ content = get_network_status_document_v3({"consensus-method": "8"}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) # excludes a footer from a version that shouldn't have it document = get_network_status_document_v3({"consensus-method": "8"}, ("directory-footer", "directory-signature")) self.assertEqual([], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) # Prior to conensus method 9 votes can still have a signature in their # footer... # # https://trac.torproject.org/7932 document = get_network_status_document_v3( { "vote-status": "vote", "consensus-methods": "1 8", }, exclude = ("directory-footer",), authorities = (get_directory_authority(is_vote = True),) ) self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())
def test_misordered_fields(self): """ Rearranges our descriptor fields. """ for is_consensus in (True, False): attr = { "vote-status": "consensus" } if is_consensus else { "vote-status": "vote" } lines = get_network_status_document_v3(attr, content=True).split("\n") for index in xrange(len(lines) - 1): # once we reach the crypto blob we're done since swapping those won't # be detected if lines[index].startswith(CRYPTO_BLOB[1:10]): break # swaps this line with the one after it test_lines = list(lines) test_lines[index], test_lines[index + 1] = test_lines[ index + 1], test_lines[index] content = "\n".join(test_lines) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) NetworkStatusDocumentV3(content, False) # constructs without validation
def test_duplicate_fields(self): """ Almost all fields can only appear once. Checking that duplicates cause validation errors. """ for is_consensus in (True, False): attr = { "vote-status": "consensus" } if is_consensus else { "vote-status": "vote" } lines = get_network_status_document_v3(attr, content=True).split("\n") for index, line in enumerate(lines): # Stop when we hit the 'directory-signature' for a couple reasons... # - that is the one field that can validly appear multiple times # - after it is a crypto blob, which won't trigger this kind of # validation failure test_lines = list(lines) if line.startswith("directory-signature "): break # duplicates the line test_lines.insert(index, line) content = "\n".join(test_lines) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) NetworkStatusDocumentV3(content, False) # constructs without validation
def test_malformed_signature(self): """ Provides malformed or missing content in the 'directory-signature' line. """ test_values = ( "", "\n", "blarg", ) for test_value in test_values: for test_attr in xrange(3): attrs = [ DOC_SIG.identity, DOC_SIG.key_digest, DOC_SIG.signature ] attrs[test_attr] = test_value content = get_network_status_document_v3( {"directory-signature": "%s %s\n%s" % tuple(attrs)}, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) NetworkStatusDocumentV3( content, False ) # checks that it's still parsable without validation
def test_misordered_fields(self): """ Rearranges our descriptor fields. """ for is_consensus in (True, False): attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} lines = get_network_status_document_v3(attr, content = True).split(b"\n") for index in xrange(len(lines) - 1): # once we reach the authority entry or later we're done since swapping # those won't be detected if is_consensus and lines[index].startswith(stem.util.str_tools._to_bytes(CRYPTO_BLOB[1:10])): break elif not is_consensus and lines[index].startswith('dir-source'): break # swaps this line with the one after it test_lines = list(lines) test_lines[index], test_lines[index + 1] = test_lines[index + 1], test_lines[index] content = b"\n".join(test_lines) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) NetworkStatusDocumentV3(content, False) # constructs without validation
def test_bandwidth_wights_malformed(self): """ Provides malformed content in the 'bandwidth-wights' line. """ test_values = ( "Wbe", "Wbe=", "Wbe=a", "Wbe=+7", ) base_weight_entry = " ".join( ["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES if e != "Wbe"]) for test_value in test_values: weight_entry = base_weight_entry.replace("Wbe=5", test_value) content = get_network_status_document_v3( {"bandwidth-weights": weight_entry}, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected, document.bandwidth_weights)
def test_voting_delay(self): """ Parses the voting-delay field. """ document = get_network_status_document_v3({"voting-delay": "12 345"}) self.assertEquals(12, document.vote_delay) self.assertEquals(345, document.dist_delay) test_values = ( "", " ", "1 a", "1\t2", "1 2.0", ) for test_value in test_values: content = get_network_status_document_v3( {"voting-delay": test_value}, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(None, document.vote_delay) self.assertEquals(None, document.dist_delay)
def test_version(self): """ Parses the network-status-version field, including trying to handle a different document version with the v3 parser. """ document = get_network_status_document_v3( {"network-status-version": "3"}) self.assertEquals(3, document.version) self.assertEquals(None, document.version_flavor) self.assertEquals(False, document.is_microdescriptor) document = get_network_status_document_v3( {"network-status-version": "3 microdesc"}) self.assertEquals(3, document.version) self.assertEquals('microdesc', document.version_flavor) self.assertEquals(True, document.is_microdescriptor) content = get_network_status_document_v3( {"network-status-version": "4"}, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(4, document.version) self.assertEquals(None, document.version_flavor) self.assertEquals(False, document.is_microdescriptor)
def test_params(self): """ General testing for the 'params' line, exercising the happy cases. """ document = get_network_status_document_v3({"params": "CircuitPriorityHalflifeMsec=30000 bwauthpid=1 unrecognized=-122"}) self.assertEquals(30000, document.params["CircuitPriorityHalflifeMsec"]) self.assertEquals(1, document.params["bwauthpid"]) self.assertEquals(-122, document.params["unrecognized"]) # empty params line content = get_network_status_document_v3({"params": ""}, content = True) document = NetworkStatusDocumentV3(content, default_params = True) self.assertEquals(DEFAULT_PARAMS, document.params) content = get_network_status_document_v3({"params": ""}, content = True) document = NetworkStatusDocumentV3(content, default_params = False) self.assertEquals({}, document.params)
def test_with_microdescriptor_router_status_entries(self): """ Includes microdescriptor flavored router status entries within the document. """ entry1 = get_router_status_entry_micro_v3({'s': 'Fast'}) entry2 = get_router_status_entry_micro_v3({ 'r': 'tornodeviennasil AcWxDFxrHetHYS5m6/MVt8ZN6AM 2013-03-13 22:09:13 78.142.142.246 443 80', 's': 'Valid', }) document = get_network_status_document_v3( {'network-status-version': '3 microdesc'}, routers=(entry1, entry2)) self.assertTrue(entry1 in document.routers.values()) self.assertTrue(entry2 in document.routers.values()) # try with an invalid RouterStatusEntry entry3 = RouterStatusEntryMicroV3( get_router_status_entry_micro_v3({'r': 'ugabuga'}, content=True), False) content = get_network_status_document_v3( {'network-status-version': '3 microdesc'}, routers=(entry3, ), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals([entry3], document.routers.values()) # try including microdescriptor entry in a normal consensus content = get_network_status_document_v3(routers=(entry1, ), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEqual([RouterStatusEntryV3(str(entry1), False)], document.routers.values())
def test_params_misordered(self): """ Check that the 'params' line is rejected if out of order. """ content = get_network_status_document_v3({"params": "unrecognized=-122 bwauthpid=1"}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False, default_params = False) self.assertEquals({"unrecognized": -122, "bwauthpid": 1}, document.params)
def test_footer_with_value(self): """ Tries to parse a descriptor with content on the 'directory-footer' line. """ content = get_network_status_document_v3({"directory-footer": "blarg"}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines())
def _readV2DirsFromCacheFile(): try: with open(MICRO_CONSENSUS_CACHE_FILE, 'rb') as f: data = f.read() consensus = NetworkStatusDocumentV3(data) return getV2DirsFromConsensus(consensus) except Exception as e: msg = ( "Failed to read cached-consensus-microdesc. Reason: {}.".format(e)) logging.debug(msg) return None
def test_with_microdescriptor_router_status_entries(self): """ Includes microdescriptor flavored router status entries within the document. """ entry1 = get_router_status_entry_micro_v3({'s': "Fast"}) entry2 = get_router_status_entry_micro_v3({'s': "Valid"}) document = get_network_status_document_v3( {"network-status-version": "3 microdesc"}, routers=(entry1, entry2)) self.assertEquals((entry1, entry2), document.routers) # try with an invalid RouterStatusEntry entry3 = RouterStatusEntryMicroV3( get_router_status_entry_micro_v3({'r': "ugabuga"}, content=True), False) content = get_network_status_document_v3( {"network-status-version": "3 microdesc"}, routers=(entry3, ), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals((entry3, ), document.routers) # try including microdescriptor entries in a normal consensus content = get_network_status_document_v3(routers=(entry1, entry2), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) expected_routers = ( RouterStatusEntryV3(str(entry1), False), RouterStatusEntryV3(str(entry2), False), ) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected_routers, document.routers)
def test_consensus_methods(self): """ Parses the consensus-methods field. """ document = get_network_status_document_v3({ "vote-status": "vote", "consensus-methods": "12 3 1 780" }) self.assertEquals([12, 3, 1, 780], document.consensus_methods) # check that we default to including consensus-method 1 content = get_network_status_document_v3({"vote-status": "vote"}, ("consensus-methods", ), content=True) document = NetworkStatusDocumentV3(content, False) self.assertEquals([1], document.consensus_methods) self.assertEquals(None, document.consensus_method) test_values = ( ("", []), (" ", []), ("1 2 3 a 5", [1, 2, 3, 5]), ("1 2 3 4.0 5", [1, 2, 3, 5]), ("2 3 4", [2, 3, 4]), # spec says version one must be included ) for test_value, expected_consensus_methods in test_values: content = get_network_status_document_v3( { "vote-status": "vote", "consensus-methods": test_value }, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected_consensus_methods, document.consensus_methods)
def test_with_router_status_entries(self): """ Includes router status entries within the document. This isn't to test the RouterStatusEntry parsing but rather the inclusion of it within the document. """ entry1 = get_router_status_entry_v3({'s': 'Fast'}) entry2 = get_router_status_entry_v3({ 'r': 'Nightfae AWt0XNId/OU2xX5xs5hVtDc5Mes 6873oEfM7fFIbxYtwllw9GPDwkA 2013-02-20 11:12:27 85.177.66.233 9001 9030', 's': 'Valid', }) document = get_network_status_document_v3(routers=(entry1, entry2)) self.assertTrue(entry1 in document.routers.values()) self.assertTrue(entry2 in document.routers.values()) # try with an invalid RouterStatusEntry entry3 = RouterStatusEntryV3( get_router_status_entry_v3({'r': 'ugabuga'}, content=True), False) content = get_network_status_document_v3(routers=(entry3, ), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals([entry3], document.routers.values()) # try including with a microdescriptor consensus content = get_network_status_document_v3( {'network-status-version': '3 microdesc'}, routers=(entry1, ), content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEqual([RouterStatusEntryMicroV3(str(entry1), False)], document.routers.values())
def test_with_directory_authorities(self): """ Includes a couple directory authorities in the document. """ for is_document_vote in (False, True): for is_authorities_vote in (False, True): authority1 = get_directory_authority({'contact': 'doctor jekyll'}, is_vote = is_authorities_vote) authority2 = get_directory_authority({'contact': 'mister hyde'}, is_vote = is_authorities_vote) vote_status = "vote" if is_document_vote else "consensus" content = get_network_status_document_v3({"vote-status": vote_status}, authorities = (authority1, authority2), content = True) if is_document_vote == is_authorities_vote: document = NetworkStatusDocumentV3(content) self.assertEquals((authority1, authority2), document.directory_authorities) else: # authority votes in a consensus or consensus authorities in a vote self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, validate = False) self.assertEquals((authority1, authority2), document.directory_authorities)
def test_bandwidth_wights_in_vote(self): """ Tries adding a 'bandwidth-wights' line to a vote. """ weight_entry = " ".join(["%s=5" % e for e in BANDWIDTH_WEIGHT_ENTRIES]) expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) content = get_network_status_document_v3({"vote-status": "vote", "bandwidth-weights": weight_entry}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected, document.bandwidth_weights)
def test_bandwidth_wights_misordered(self): """ Check that the 'bandwidth-wights' line is rejected if out of order. """ weight_entry = " ".join(["%s=5" % e for e in reversed(BANDWIDTH_WEIGHT_ENTRIES)]) expected = dict([(e, 5) for e in BANDWIDTH_WEIGHT_ENTRIES]) content = get_network_status_document_v3({"bandwidth-weights": weight_entry}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(expected, document.bandwidth_weights)
def test_missing_fields(self): """ Excludes mandatory fields from both a vote and consensus document. """ for is_consensus in (True, False): attr = {"vote-status": "consensus"} if is_consensus else {"vote-status": "vote"} is_vote = not is_consensus for entries in (HEADER_STATUS_DOCUMENT_FIELDS, FOOTER_STATUS_DOCUMENT_FIELDS): for field, in_votes, in_consensus, is_mandatory in entries: if is_mandatory and ((is_consensus and in_consensus) or (is_vote and in_votes)): content = get_network_status_document_v3(attr, exclude = (field,), content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) NetworkStatusDocumentV3(content, False) # constructs without validation
def test_flag_thresholds(self): """ Parses the flag-thresholds entry. """ test_values = ( ("", {}), ("fast-speed=40960", {u"fast-speed": 40960}), # numeric value ("guard-wfu=94.669%", {u"guard-wfu": 0.94669}), # percentage value ("guard-wfu=94.669% guard-tk=691200", {u"guard-wfu": 0.94669, u"guard-tk": 691200}), # multiple values ) for test_value, expected_value in test_values: document = get_network_status_document_v3({"vote-status": "vote", "flag-thresholds": test_value}) self.assertEquals(expected_value, document.flag_thresholds) # parses a full entry found in an actual vote full_line = "stable-uptime=693369 stable-mtbf=153249 fast-speed=40960 guard-wfu=94.669% guard-tk=691200 guard-bw-inc-exits=174080 guard-bw-exc-exits=184320 enough-mtbf=1" expected_value = { u"stable-uptime": 693369, u"stable-mtbf": 153249, u"fast-speed": 40960, u"guard-wfu": 0.94669, u"guard-tk": 691200, u"guard-bw-inc-exits": 174080, u"guard-bw-exc-exits": 184320, u"enough-mtbf": 1, } document = get_network_status_document_v3({"vote-status": "vote", "flag-thresholds": full_line}) self.assertEquals(expected_value, document.flag_thresholds) test_values = ( "stable-uptime 693369", # not a key=value mapping "stable-uptime=a693369", # non-numeric value "guard-wfu=94.669%%", # double quote "stable-uptime=693369\tstable-mtbf=153249", # non-space divider ) for test_value in test_values: content = get_network_status_document_v3({"vote-status": "vote", "flag-thresholds": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals({}, document.flag_thresholds)
def test_authority_validation_flag_propagation(self): """ Includes invalid certificate content in an authority entry. This is testing that the 'validate' flag propagages from the document to authority, and authority to certificate classes. """ # make the dir-key-published field of the certiciate be malformed authority_content = get_directory_authority(is_vote = True, content = True) authority_content = authority_content.replace(b"dir-key-published 2011", b"dir-key-published 2011a") authority = DirectoryAuthority(authority_content, False, True) content = get_network_status_document_v3({"vote-status": "vote"}, authorities = (authority,), content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, validate = False) self.assertEquals((authority,), document.directory_authorities)
def _processConsensus(self, raw): '''Decompress consensus, parse, write to "cached-consensus" and choose a new set of endpoints to use for the next download. .. note: This is run in a separate worker thread using twisted.internet.threads.deferToThread() because consensus parsing can take a while. :param str raw: compressed consensus bytes :returns: stem.descriptors.networkstatus.NetworkStatusDocumentV3 ''' raw = zlib.decompress(raw) consensus = NetworkStatusDocumentV3(raw) self._cacheConsensus(consensus) logging.debug("Wrote cached-consensus.") self._endpoints = self._extractV2DirEndpoints(consensus) logging.debug("Found {} V2Dir endpoints.".format(len(self._endpoints))) return consensus
def test_params_malformed(self): """ Parses a 'params' line with malformed content. """ test_values = ( "foo=", "foo=abc", "foo=+123", "foo=12\tbar=12", ) for test_value in test_values: content = get_network_status_document_v3({"params": test_value}, content = True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(DEFAULT_PARAMS, document.params)
def test_time_fields(self): """ Parses invalid published, valid-after, fresh-until, and valid-until fields. All are simply datetime values. """ expected = datetime.datetime(2012, 9, 2, 22, 0, 0) test_value = "2012-09-02 22:00:00" document = get_network_status_document_v3({ "vote-status": "vote", "published": test_value, "valid-after": test_value, "fresh-until": test_value, "valid-until": test_value, }) self.assertEquals(expected, document.published) self.assertEquals(expected, document.valid_after) self.assertEquals(expected, document.fresh_until) self.assertEquals(expected, document.valid_until) test_values = ( "", " ", "2012-12-12", "2012-12-12 01:01:", "2012-12-12 01:a1:01", ) for field in ('published', 'valid-after', 'fresh-until', 'valid-until'): attr = field.replace('-', '_') for test_value in test_values: content = get_network_status_document_v3( { "vote-status": "vote", field: test_value }, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEquals(None, getattr(document, attr))
def refresh(self): """ Attempt to refresh the consensus with the latest one available. """ from onionbalance.hs_v3.onionbalance import my_onionbalance # Fetch the current md consensus from the control port md_consensus_str = my_onionbalance.controller.get_md_consensus().encode() try: self.consensus = NetworkStatusDocumentV3(md_consensus_str) except ValueError: logger.warning("No valid consensus received. Waiting for one...") return # Check if it's live if not self.is_live(): logger.info("Loaded consensus is not live. Waiting for a live one.") return self.nodes = self._initialize_nodes()
def test_footer_consensus_method_requirement(self): """ Check that validation will notice if a footer appears before it was introduced. """ content = get_network_status_document_v3({"consensus-method": "8"}, content=True) self.assertRaises(ValueError, NetworkStatusDocumentV3, content) document = NetworkStatusDocumentV3(content, False) self.assertEqual([DOC_SIG], document.signatures) self.assertEqual([], document.get_unrecognized_lines()) # excludes a footer from a version that shouldn't have it document = get_network_status_document_v3( {"consensus-method": "8"}, ("directory-footer", "directory-signature")) self.assertEqual([], document.signatures) self.assertEqual([], document.get_unrecognized_lines())
def _processRawMicroconsensus(raw): raw = zlib.decompress(raw) consensus = NetworkStatusDocumentV3(raw) _writeConsensusCacheFile(consensus) return consensus