def test_parse_descriptors_parseServerDescriptorsFile(self):
"""Test for ``b.p.descriptors.parseServerDescriptorsFile``."""
descFile = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR)
routers = descriptors.parseServerDescriptorsFile(descFile)
self.assertIsInstance(routers, list)
bridge = routers[0]
self.assertIsInstance(bridge, RelayDescriptor)
self.assertEqual(bridge.address, self.expectedIPBridge0)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseServerDescriptorsFile(self):
"""Test for ``b.p.descriptors.parseServerDescriptorsFile``."""
descFile = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR)
routers = descriptors.parseServerDescriptorsFile(descFile)
self.assertIsInstance(routers, list)
bridge = routers[0]
self.assertIsInstance(bridge, RelayDescriptor)
self.assertEqual(bridge.address, self.expectedIPBridge0)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseExtraInfoFiles_ed25519(self):
"""Test parsing an extrainfo descriptor with Ed25519 keys/certificates.
"""
descFileOne = io.BytesIO(BRIDGE_SERVER_DESCRIPTOR_ED25519)
routers = descriptors.parseServerDescriptorsFile(descFileOne)
self.assertIsInstance(routers, list)
self.assertEqual(len(routers), 1)
bridge = routers[0]
self.assertIsInstance(bridge, RelayDescriptor)
self.assertEqual(bridge.address, u'80.92.79.70')
self.assertEqual(bridge.fingerprint, u'312D64274C29156005843EECB19C6865FA3CC10C')
def load(state, hashring, clear=False):
"""Read and parse all descriptors, and load into a bridge hashring.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.hashring`` instance. The ``state`` will be
saved again at the end of this function.
:type hashring: :class:`~bridgedb.bridgerings.BridgeSplitter`
:param hashring: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
hashring before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
hashring.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
for auth in state.BRIDGE_AUTHORITY_DIRECTORIES:
logging.info("Processing descriptors in %s directory..." % auth)
bridges = {}
timestamps = {}
fn = expandBridgeAuthDir(auth, state.STATUS_FILE)
logging.info("Opening networkstatus file: %s" % fn)
networkstatuses = descriptors.parseNetworkStatusFile(fn)
logging.debug("Closing networkstatus file: %s" % fn)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
fn = expandBridgeAuthDir(auth, filename)
logging.info("Opening bridge-server-descriptor file: '%s'" % fn)
serverdescriptors = descriptors.parseServerDescriptorsFile(fn)
logging.debug("Closing bridge-server-descriptor file: '%s'" % fn)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn((
"Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router,
ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
eifiles = [
expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES
]
extrainfos = descriptors.parseExtraInfoFiles(*eifiles)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(
("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.") %
router.fingerprint)
blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE)
inserted = 0
logging.info(
"Trying to insert %d bridges into hashring, %d of which "
"have the 'Running' flag..." %
(len(bridges),
len(list(filter(lambda b: b.flags.running, bridges.values())))))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn(
"Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
# Skip insertion of blacklisted bridges.
elif bridge in blacklist.keys():
logging.warn(
"Not distributing blacklisted Bridge %s %s:%s: %s" %
(bridge, bridge.address, bridge.orPort, blacklist[bridge]))
# Skip bridges that are running a blacklisted version of Tor.
elif bridge.runsVersion(state.BLACKLISTED_TOR_VERSIONS):
logging.warn(
"Not distributing bridge %s because it runs blacklisted "
"Tor version %s." % (router.fingerprint, bridge.software))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
hashring.insert(bridge)
inserted += 1
logging.info("Tried to insert %d bridges into hashring. Resulting "
"hashring is of length %d." % (inserted, len(hashring)))
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()
def load(state, hashring, clear=False):
"""Read and parse all descriptors, and load into a bridge hashring.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.hashring`` instance. The ``state`` will be
saved again at the end of this function.
:type hashring: :class:`~bridgedb.Bridges.BridgeSplitter`
:param hashring: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
hashring before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
hashring.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
for auth in state.BRIDGE_AUTHORITY_DIRECTORIES:
logging.info("Processing descriptors in %s directory..." % auth)
bridges = {}
timestamps = {}
fn = expandBridgeAuthDir(auth, state.STATUS_FILE)
logging.info("Opening networkstatus file: %s" % fn)
networkstatuses = descriptors.parseNetworkStatusFile(fn)
logging.debug("Closing networkstatus file: %s" % fn)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
fn = expandBridgeAuthDir(auth, filename)
logging.info("Opening bridge-server-descriptor file: '%s'" % fn)
serverdescriptors = descriptors.parseServerDescriptorsFile(fn)
logging.debug("Closing bridge-server-descriptor file: '%s'" % fn)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn(
("Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router, ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES]
extrainfos = descriptors.parseExtraInfoFiles(*eifiles)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.")
% router.fingerprint)
blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE)
inserted = 0
logging.info("Inserting %d bridges into hashring..." % len(bridges))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn("Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
# Skip insertion of blacklisted bridges.
elif bridge in blacklist.keys():
logging.warn("Not distributing blacklisted Bridge %s %s:%s: %s" %
(bridge, bridge.address, bridge.orPort, blacklist[bridge]))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
hashring.insert(bridge)
inserted += 1
logging.info("Done inserting %d bridges into hashring." % inserted)
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()
def load(state, splitter, clear=False):
"""Read and parse all descriptors, and load into a bridge splitter.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.splitter`` instance. The ``state`` will be
saved again at the end of this function.
:type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>`
:param splitter: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
splitter before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.Main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
splitter.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
bridges = {}
timestamps = {}
logging.info("Opening networkstatus file: %s" % state.STATUS_FILE)
networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE)
logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
logging.info("Opening bridge-server-descriptor file: '%s'" % filename)
serverdescriptors = descriptors.parseServerDescriptorsFile(filename)
logging.debug("Closing bridge-server-descriptor file: '%s'" % filename)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn(
("Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router, ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.") %
router.fingerprint)
inserted = 0
logging.info("Inserting %d bridges into splitter..." % len(bridges))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn(
"Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
splitter.insert(bridge)
inserted += 1
logging.info("Done inserting %d bridges into splitter." % inserted)
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()