def test_parse_descriptors_parseServerDescriptorsFile(self): """Test for ``b.p.descriptors.parseServerDescriptorsFile``.""" descFile = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR) routers = descriptors.parseServerDescriptorsFile(descFile) self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) self.assertEqual(bridge.address, self.expectedIPBridge0) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseServerDescriptorsFile(self): """Test for ``b.p.descriptors.parseServerDescriptorsFile``.""" descFile = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR) routers = descriptors.parseServerDescriptorsFile(descFile) self.assertIsInstance(routers, list) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) self.assertEqual(bridge.address, self.expectedIPBridge0) self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseExtraInfoFiles_ed25519(self): """Test parsing an extrainfo descriptor with Ed25519 keys/certificates. """ descFileOne = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR_ED25519) routers = descriptors.parseServerDescriptorsFile(descFileOne) self.assertIsInstance(routers, list) self.assertEqual(len(routers), 1) bridge = routers[0] self.assertIsInstance(bridge, RelayDescriptor) self.assertEqual(bridge.address, u'80.92.79.70') self.assertEqual(bridge.fingerprint, u'312D64274C29156005843EECB19C6865FA3CC10C')
def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. :type hashring: :class:`~bridgedb.bridgerings.BridgeSplitter` :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") hashring.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: logging.info("Processing descriptors in %s directory..." % auth) bridges = {} timestamps = {} fn = expandBridgeAuthDir(auth, state.STATUS_FILE) logging.info("Opening networkstatus file: %s" % fn) networkstatuses = descriptors.parseNetworkStatusFile(fn) logging.debug("Closing networkstatus file: %s" % fn) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: fn = expandBridgeAuthDir(auth, filename) logging.info("Opening bridge-server-descriptor file: '%s'" % fn) serverdescriptors = descriptors.parseServerDescriptorsFile(fn) logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn(( "Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] eifiles = [ expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES ] extrainfos = descriptors.parseExtraInfoFiles(*eifiles) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn( ("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE) inserted = 0 logging.info( "Trying to insert %d bridges into hashring, %d of which " "have the 'Running' flag..." % (len(bridges), len(list(filter(lambda b: b.flags.running, bridges.values()))))) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn( "Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) # Skip insertion of blacklisted bridges. elif bridge in blacklist.keys(): logging.warn( "Not distributing blacklisted Bridge %s %s:%s: %s" % (bridge, bridge.address, bridge.orPort, blacklist[bridge])) # Skip bridges that are running a blacklisted version of Tor. elif bridge.runsVersion(state.BLACKLISTED_TOR_VERSIONS): logging.warn( "Not distributing bridge %s because it runs blacklisted " "Tor version %s." % (router.fingerprint, bridge.software)) else: # If the bridge is not running, then it is skipped during the # insertion process. hashring.insert(bridge) inserted += 1 logging.info("Tried to insert %d bridges into hashring. Resulting " "hashring is of length %d." % (inserted, len(hashring))) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()
def load(state, hashring, clear=False): """Read and parse all descriptors, and load into a bridge hashring. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.hashring`` instance. The ``state`` will be saved again at the end of this function. :type hashring: :class:`~bridgedb.Bridges.BridgeSplitter` :param hashring: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the hashring before parsing for new ones. """ if not state: logging.fatal("bridgedb.main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") hashring.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") for auth in state.BRIDGE_AUTHORITY_DIRECTORIES: logging.info("Processing descriptors in %s directory..." % auth) bridges = {} timestamps = {} fn = expandBridgeAuthDir(auth, state.STATUS_FILE) logging.info("Opening networkstatus file: %s" % fn) networkstatuses = descriptors.parseNetworkStatusFile(fn) logging.debug("Closing networkstatus file: %s" % fn) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: fn = expandBridgeAuthDir(auth, filename) logging.info("Opening bridge-server-descriptor file: '%s'" % fn) serverdescriptors = descriptors.parseServerDescriptorsFile(fn) logging.debug("Closing bridge-server-descriptor file: '%s'" % fn) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn( ("Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES] extrainfos = descriptors.parseExtraInfoFiles(*eifiles) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn(("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE) inserted = 0 logging.info("Inserting %d bridges into hashring..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn("Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) # Skip insertion of blacklisted bridges. elif bridge in blacklist.keys(): logging.warn("Not distributing blacklisted Bridge %s %s:%s: %s" % (bridge, bridge.address, bridge.orPort, blacklist[bridge])) else: # If the bridge is not running, then it is skipped during the # insertion process. hashring.insert(bridge) inserted += 1 logging.info("Done inserting %d bridges into hashring." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()
def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.splitter`` instance. The ``state`` will be saved again at the end of this function. :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` :param splitter: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the splitter before parsing for new ones. """ if not state: logging.fatal("bridgedb.Main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") splitter.clear() logging.info("Loading bridges...") ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS if ignoreNetworkstatus: logging.info("Ignoring BridgeAuthority networkstatus documents.") bridges = {} timestamps = {} logging.info("Opening networkstatus file: %s" % state.STATUS_FILE) networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE) logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE) logging.info("Processing networkstatus descriptors...") for router in networkstatuses: bridge = Bridge() bridge.updateFromNetworkStatus(router, ignoreNetworkstatus) try: bridge.assertOK() except MalformedBridgeInfo as error: logging.warn(str(error)) else: bridges[bridge.fingerprint] = bridge for filename in state.BRIDGE_FILES: logging.info("Opening bridge-server-descriptor file: '%s'" % filename) serverdescriptors = descriptors.parseServerDescriptorsFile(filename) logging.debug("Closing bridge-server-descriptor file: '%s'" % filename) for router in serverdescriptors: try: bridge = bridges[router.fingerprint] except KeyError: logging.warn( ("Received server descriptor for bridge '%s' which wasn't " "in the networkstatus!") % router.fingerprint) if ignoreNetworkstatus: bridge = Bridge() else: continue try: bridge.updateFromServerDescriptor(router, ignoreNetworkstatus) except (ServerDescriptorWithoutNetworkstatus, MissingServerDescriptorDigest, ServerDescriptorDigestMismatch) as error: logging.warn(str(error)) # Reject any routers whose server descriptors didn't pass # :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those # bridges who don't have corresponding networkstatus # documents, or whose server descriptor digests don't check # out: bridges.pop(router.fingerprint) continue if state.COLLECT_TIMESTAMPS: # Update timestamps from server descriptors, not from network # status descriptors (because networkstatus documents and # descriptors aren't authenticated in any way): if bridge.fingerprint in timestamps.keys(): timestamps[bridge.fingerprint].append(router.published) else: timestamps[bridge.fingerprint] = [router.published] extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES) for fingerprint, router in extrainfos.items(): try: bridges[fingerprint].updateFromExtraInfoDescriptor(router) except MalformedBridgeInfo as error: logging.warn(str(error)) except KeyError as error: logging.warn(("Received extrainfo descriptor for bridge '%s', " "but could not find bridge with that fingerprint.") % router.fingerprint) inserted = 0 logging.info("Inserting %d bridges into splitter..." % len(bridges)) for fingerprint, bridge in bridges.items(): # Skip insertion of bridges which are geolocated to be in one of the # NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute # bridges from: if bridge.country in state.NO_DISTRIBUTION_COUNTRIES: logging.warn( "Not distributing Bridge %s %s:%s in country %s!" % (bridge, bridge.address, bridge.orPort, bridge.country)) else: # If the bridge is not running, then it is skipped during the # insertion process. splitter.insert(bridge) inserted += 1 logging.info("Done inserting %d bridges into splitter." % inserted) if state.COLLECT_TIMESTAMPS: reactor.callInThread(updateBridgeHistory, bridges, timestamps) state.save()