def test_parse_descriptors_parseNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) self.assertIsInstance(routers, list)
def test_parse_descriptors_parseNetworkStatusFile_HSDir_flag(self): """A Bridge networkstatus descriptor with the HSDir flag should be possible to parse (without errors), however, the flag should be ignored (since the :class:`bridgedb.bridges.Flags` class doesn't care about it). See also: :trac:`16616` """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( b's Fast Guard Running Stable Valid', b's Fast Guard Running Stable Valid HSDir') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', unparseable) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] for flag in [ u'Fast', u'Guard', u'Running', u'Stable', u'Valid', u'HSDir' ]: self.assertTrue(flag in bridge.flags, ("Expected to parse the %r flag from a bridge " "networkstatus document, but the flag was not " "found!")) self.assertTrue(self.removeTestDescriptorsFile(descFile))
def test_parse_descriptors_parseNetworkStatusFile_return_type(self): """``b.p.descriptors.parseNetworkStatusFile`` should return a dict.""" # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) self.assertIsInstance(routers, list)
def test_parse_descriptors_parseNetworkStatusFile_one_file(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge networkstatus descriptor. """ # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertEqual(bridge.address, self.expectedIPBridge0) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_has_RouterStatusEntryV2(self): """The items in the dict returned from ``b.p.descriptors.parseNetworkStatusFile`` should be ``RouterStatusEntryV2``s. """ # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertIsInstance(bridge, RouterStatusEntryV3)
def test_parse_descriptors_parseNetworkStatusFile_one_file(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge networkstatus descriptor. """ # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertEqual(bridge.address, self.expectedIPBridge0) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_has_RouterStatusEntryV2(self): """The items in the dict returned from ``b.p.descriptors.parseNetworkStatusFile`` should be ``RouterStatusEntryV2``s. """ # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', BRIDGE_NETWORKSTATUS_0) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertIsInstance(bridge, RouterStatusEntryV3)
def test_parse_descriptors_parseNetworkStatusFile_with_annotations(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with some document headers before the first 'r'-line. """ expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' with open(descFile, 'w') as fh: fh.write('signature and stuff from the BridgeAuth would go here\n') fh.write('some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertIn(bridge.address, expectedIPs) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_with_annotations(self): """Test ``b.p.descriptors.parseNetworkStatusFile`` with some document headers before the first 'r'-line. """ expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1] descFile = 'networkstatus-bridges' with open(descFile, 'w') as fh: fh.write('signature and stuff from the BridgeAuth would go here\n') fh.write('some more annotations with parameters and stuff\n') fh.write(BRIDGE_NETWORKSTATUS_0) fh.write(BRIDGE_NETWORKSTATUS_1) fh.flush() routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] self.assertIn(bridge.address, expectedIPs) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_HSDir_flag(self): """A Bridge networkstatus descriptor with the HSDir flag should be possible to parse (without errors), however, the flag should be ignored (since the :class:`bridgedb.bridges.Flags` class doesn't care about it). See also: :trac:`16616` """ unparseable = BRIDGE_NETWORKSTATUS_0.replace( 's Fast Guard Running Stable Valid', 's Fast Guard Running Stable Valid HSDir') # Write the descriptor to a file for testing. This is necessary # because the function opens the networkstatus file to read it. descFile = self.writeTestDescriptorsToFile('networkstatus-bridges', unparseable) routers = descriptors.parseNetworkStatusFile(descFile) bridge = routers[0] for flag in [u'Fast', u'Guard', u'Running', u'Stable', u'Valid', u'HSDir']: self.assertTrue(flag in bridge.flags, ("Expected to parse the %r flag from a bridge " "networkstatus document, but the flag was not " "found!"))
def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. :type hashring: :class:`~bridgedb.bridgerings.BridgeSplitter` :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") hashring.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: logging.info("Processing descriptors in %s directory..." % auth) bridges = {} timestamps = {} fn = expandBridgeAuthDir(auth, state.STATUS_FILE) logging.info("Opening networkstatus file: %s" % fn) networkstatuses = descriptors.parseNetworkStatusFile(fn) logging.debug("Closing networkstatus file: %s" % fn) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: fn = expandBridgeAuthDir(auth, filename) logging.info("Opening bridge-server-descriptor file: '%s'" % fn) serverdescriptors = descriptors.parseServerDescriptorsFile(fn) logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn(( "Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] eifiles = [ expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES ] extrainfos = descriptors.parseExtraInfoFiles(*eifiles) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn( ("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE) inserted = 0 logging.info( "Trying to insert %d bridges into hashring, %d of which " "have the 'Running' flag..." % (len(bridges), len(list(filter(lambda b: b.flags.running, bridges.values()))))) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn( "Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) # Skip insertion of blacklisted bridges. elif bridge in blacklist.keys(): logging.warn( "Not distributing blacklisted Bridge %s %s:%s: %s" % (bridge, bridge.address, bridge.orPort, blacklist[bridge])) # Skip bridges that are running a blacklisted version of Tor. elif bridge.runsVersion(state.BLACKLISTED_TOR_VERSIONS): logging.warn( "Not distributing bridge %s because it runs blacklisted " "Tor version %s." % (router.fingerprint, bridge.software)) else: # If the bridge is not running, then it is skipped during the # insertion process. hashring.insert(bridge) inserted += 1 logging.info("Tried to insert %d bridges into hashring. Resulting " "hashring is of length %d." % (inserted, len(hashring))) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()
def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. :type hashring: :class:`~bridgedb.Bridges.BridgeSplitter` :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") hashring.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: logging.info("Processing descriptors in %s directory..." % auth) bridges = {} timestamps = {} fn = expandBridgeAuthDir(auth, state.STATUS_FILE) logging.info("Opening networkstatus file: %s" % fn) networkstatuses = descriptors.parseNetworkStatusFile(fn) logging.debug("Closing networkstatus file: %s" % fn) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: fn = expandBridgeAuthDir(auth, filename) logging.info("Opening bridge-server-descriptor file: '%s'" % fn) serverdescriptors = descriptors.parseServerDescriptorsFile(fn) logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn( ("Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES] extrainfos = descriptors.parseExtraInfoFiles(*eifiles) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn(("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE) inserted = 0 logging.info("Inserting %d bridges into hashring..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn("Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) # Skip insertion of blacklisted bridges. elif bridge in blacklist.keys(): logging.warn("Not distributing blacklisted Bridge %s %s:%s: %s" % (bridge, bridge.address, bridge.orPort, blacklist[bridge])) else: # If the bridge is not running, then it is skipped during the # insertion process. hashring.insert(bridge) inserted += 1 logging.info("Done inserting %d bridges into hashring." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()
def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.splitter`` instance. The ``state`` will be saved again at the end of this function. :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` :param splitter: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the splitter before parsing for new ones. """ if not state: logging.fatal("bridgedb.Main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") splitter.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") bridges = {} timestamps = {} logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: logging.info("Opening bridge-server-descriptor file: '%s'" % filename) serverdescriptors = descriptors.parseServerDescriptorsFile(filename) logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn( ("Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn(("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) inserted = 0 logging.info("Inserting %d bridges into splitter..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn( "Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) else: # If the bridge is not running, then it is skipped during the # insertion process. splitter.insert(bridge) inserted += 1 logging.info("Done inserting %d bridges into splitter." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()