def test_parse_descriptors_parseNetworkStatusFile_return_type(self):
"""``b.p.descriptors.parseNetworkStatusFile`` should return a dict."""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
self.assertIsInstance(routers, list)
def test_parse_descriptors_parseNetworkStatusFile_HSDir_flag(self):
"""A Bridge networkstatus descriptor with the HSDir flag should be
possible to parse (without errors), however, the flag should be ignored
(since the :class:`bridgedb.bridges.Flags` class doesn't care about it).
See also: :trac:`16616`
"""
unparseable = BRIDGE_NETWORKSTATUS_0.replace(
b's Fast Guard Running Stable Valid',
b's Fast Guard Running Stable Valid HSDir')
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
unparseable)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
for flag in [
u'Fast', u'Guard', u'Running', u'Stable', u'Valid', u'HSDir'
]:
self.assertTrue(flag in bridge.flags,
("Expected to parse the %r flag from a bridge "
"networkstatus document, but the flag was not "
"found!"))
self.assertTrue(self.removeTestDescriptorsFile(descFile))
def test_parse_descriptors_parseNetworkStatusFile_return_type(self):
"""``b.p.descriptors.parseNetworkStatusFile`` should return a dict."""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
self.assertIsInstance(routers, list)
def test_parse_descriptors_parseNetworkStatusFile_one_file(self):
"""Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge
networkstatus descriptor.
"""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertEqual(bridge.address, self.expectedIPBridge0)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_has_RouterStatusEntryV2(self):
"""The items in the dict returned from
``b.p.descriptors.parseNetworkStatusFile`` should be
``RouterStatusEntryV2``s.
"""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile("networkstatus-bridges", BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertIsInstance(bridge, RouterStatusEntryV3)
def test_parse_descriptors_parseNetworkStatusFile_one_file(self):
"""Test ``b.p.descriptors.parseNetworkStatusFile`` with one bridge
networkstatus descriptor.
"""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertEqual(bridge.address, self.expectedIPBridge0)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_has_RouterStatusEntryV2(self):
"""The items in the dict returned from
``b.p.descriptors.parseNetworkStatusFile`` should be
``RouterStatusEntryV2``s.
"""
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
BRIDGE_NETWORKSTATUS_0)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertIsInstance(bridge, RouterStatusEntryV3)
def test_parse_descriptors_parseNetworkStatusFile_with_annotations(self):
"""Test ``b.p.descriptors.parseNetworkStatusFile`` with some document
headers before the first 'r'-line.
"""
expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1]
descFile = 'networkstatus-bridges'
with open(descFile, 'w') as fh:
fh.write('signature and stuff from the BridgeAuth would go here\n')
fh.write('some more annotations with parameters and stuff\n')
fh.write(BRIDGE_NETWORKSTATUS_0)
fh.write(BRIDGE_NETWORKSTATUS_1)
fh.flush()
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertIn(bridge.address, expectedIPs)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_with_annotations(self):
"""Test ``b.p.descriptors.parseNetworkStatusFile`` with some document
headers before the first 'r'-line.
"""
expectedIPs = [self.expectedIPBridge0, self.expectedIPBridge1]
descFile = 'networkstatus-bridges'
with open(descFile, 'w') as fh:
fh.write('signature and stuff from the BridgeAuth would go here\n')
fh.write('some more annotations with parameters and stuff\n')
fh.write(BRIDGE_NETWORKSTATUS_0)
fh.write(BRIDGE_NETWORKSTATUS_1)
fh.flush()
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
self.assertIn(bridge.address, expectedIPs)
self.assertEqual(bridge.fingerprint, self.expectedFprBridge0)
def test_parse_descriptors_parseNetworkStatusFile_HSDir_flag(self):
"""A Bridge networkstatus descriptor with the HSDir flag should be
possible to parse (without errors), however, the flag should be ignored
(since the :class:`bridgedb.bridges.Flags` class doesn't care about it).
See also: :trac:`16616`
"""
unparseable = BRIDGE_NETWORKSTATUS_0.replace(
's Fast Guard Running Stable Valid',
's Fast Guard Running Stable Valid HSDir')
# Write the descriptor to a file for testing. This is necessary
# because the function opens the networkstatus file to read it.
descFile = self.writeTestDescriptorsToFile('networkstatus-bridges',
unparseable)
routers = descriptors.parseNetworkStatusFile(descFile)
bridge = routers[0]
for flag in [u'Fast', u'Guard', u'Running',
u'Stable', u'Valid', u'HSDir']:
self.assertTrue(flag in bridge.flags,
("Expected to parse the %r flag from a bridge "
"networkstatus document, but the flag was not "
"found!"))
def load(state, hashring, clear=False):
"""Read and parse all descriptors, and load into a bridge hashring.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.hashring`` instance. The ``state`` will be
saved again at the end of this function.
:type hashring: :class:`~bridgedb.bridgerings.BridgeSplitter`
:param hashring: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
hashring before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
hashring.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
for auth in state.BRIDGE_AUTHORITY_DIRECTORIES:
logging.info("Processing descriptors in %s directory..." % auth)
bridges = {}
timestamps = {}
fn = expandBridgeAuthDir(auth, state.STATUS_FILE)
logging.info("Opening networkstatus file: %s" % fn)
networkstatuses = descriptors.parseNetworkStatusFile(fn)
logging.debug("Closing networkstatus file: %s" % fn)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
fn = expandBridgeAuthDir(auth, filename)
logging.info("Opening bridge-server-descriptor file: '%s'" % fn)
serverdescriptors = descriptors.parseServerDescriptorsFile(fn)
logging.debug("Closing bridge-server-descriptor file: '%s'" % fn)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn((
"Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router,
ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
eifiles = [
expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES
]
extrainfos = descriptors.parseExtraInfoFiles(*eifiles)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(
("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.") %
router.fingerprint)
blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE)
inserted = 0
logging.info(
"Trying to insert %d bridges into hashring, %d of which "
"have the 'Running' flag..." %
(len(bridges),
len(list(filter(lambda b: b.flags.running, bridges.values())))))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn(
"Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
# Skip insertion of blacklisted bridges.
elif bridge in blacklist.keys():
logging.warn(
"Not distributing blacklisted Bridge %s %s:%s: %s" %
(bridge, bridge.address, bridge.orPort, blacklist[bridge]))
# Skip bridges that are running a blacklisted version of Tor.
elif bridge.runsVersion(state.BLACKLISTED_TOR_VERSIONS):
logging.warn(
"Not distributing bridge %s because it runs blacklisted "
"Tor version %s." % (router.fingerprint, bridge.software))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
hashring.insert(bridge)
inserted += 1
logging.info("Tried to insert %d bridges into hashring. Resulting "
"hashring is of length %d." % (inserted, len(hashring)))
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()
def load(state, hashring, clear=False):
"""Read and parse all descriptors, and load into a bridge hashring.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.hashring`` instance. The ``state`` will be
saved again at the end of this function.
:type hashring: :class:`~bridgedb.Bridges.BridgeSplitter`
:param hashring: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
hashring before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
hashring.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
for auth in state.BRIDGE_AUTHORITY_DIRECTORIES:
logging.info("Processing descriptors in %s directory..." % auth)
bridges = {}
timestamps = {}
fn = expandBridgeAuthDir(auth, state.STATUS_FILE)
logging.info("Opening networkstatus file: %s" % fn)
networkstatuses = descriptors.parseNetworkStatusFile(fn)
logging.debug("Closing networkstatus file: %s" % fn)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
fn = expandBridgeAuthDir(auth, filename)
logging.info("Opening bridge-server-descriptor file: '%s'" % fn)
serverdescriptors = descriptors.parseServerDescriptorsFile(fn)
logging.debug("Closing bridge-server-descriptor file: '%s'" % fn)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn(
("Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router, ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
eifiles = [expandBridgeAuthDir(auth, fn) for fn in state.EXTRA_INFO_FILES]
extrainfos = descriptors.parseExtraInfoFiles(*eifiles)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.")
% router.fingerprint)
blacklist = parseBridgeBlacklistFile(state.NO_DISTRIBUTION_FILE)
inserted = 0
logging.info("Inserting %d bridges into hashring..." % len(bridges))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn("Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
# Skip insertion of blacklisted bridges.
elif bridge in blacklist.keys():
logging.warn("Not distributing blacklisted Bridge %s %s:%s: %s" %
(bridge, bridge.address, bridge.orPort, blacklist[bridge]))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
hashring.insert(bridge)
inserted += 1
logging.info("Done inserting %d bridges into hashring." % inserted)
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()
def load(state, splitter, clear=False):
"""Read and parse all descriptors, and load into a bridge splitter.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.splitter`` instance. The ``state`` will be
saved again at the end of this function.
:type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>`
:param splitter: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
splitter before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.Main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
splitter.clear()
logging.info("Loading bridges...")
ignoreNetworkstatus = state.IGNORE_NETWORKSTATUS
if ignoreNetworkstatus:
logging.info("Ignoring BridgeAuthority networkstatus documents.")
bridges = {}
timestamps = {}
logging.info("Opening networkstatus file: %s" % state.STATUS_FILE)
networkstatuses = descriptors.parseNetworkStatusFile(state.STATUS_FILE)
logging.debug("Closing networkstatus file: %s" % state.STATUS_FILE)
logging.info("Processing networkstatus descriptors...")
for router in networkstatuses:
bridge = Bridge()
bridge.updateFromNetworkStatus(router, ignoreNetworkstatus)
try:
bridge.assertOK()
except MalformedBridgeInfo as error:
logging.warn(str(error))
else:
bridges[bridge.fingerprint] = bridge
for filename in state.BRIDGE_FILES:
logging.info("Opening bridge-server-descriptor file: '%s'" % filename)
serverdescriptors = descriptors.parseServerDescriptorsFile(filename)
logging.debug("Closing bridge-server-descriptor file: '%s'" % filename)
for router in serverdescriptors:
try:
bridge = bridges[router.fingerprint]
except KeyError:
logging.warn(
("Received server descriptor for bridge '%s' which wasn't "
"in the networkstatus!") % router.fingerprint)
if ignoreNetworkstatus:
bridge = Bridge()
else:
continue
try:
bridge.updateFromServerDescriptor(router, ignoreNetworkstatus)
except (ServerDescriptorWithoutNetworkstatus,
MissingServerDescriptorDigest,
ServerDescriptorDigestMismatch) as error:
logging.warn(str(error))
# Reject any routers whose server descriptors didn't pass
# :meth:`~bridges.Bridge._checkServerDescriptor`, i.e. those
# bridges who don't have corresponding networkstatus
# documents, or whose server descriptor digests don't check
# out:
bridges.pop(router.fingerprint)
continue
if state.COLLECT_TIMESTAMPS:
# Update timestamps from server descriptors, not from network
# status descriptors (because networkstatus documents and
# descriptors aren't authenticated in any way):
if bridge.fingerprint in timestamps.keys():
timestamps[bridge.fingerprint].append(router.published)
else:
timestamps[bridge.fingerprint] = [router.published]
extrainfos = descriptors.parseExtraInfoFiles(*state.EXTRA_INFO_FILES)
for fingerprint, router in extrainfos.items():
try:
bridges[fingerprint].updateFromExtraInfoDescriptor(router)
except MalformedBridgeInfo as error:
logging.warn(str(error))
except KeyError as error:
logging.warn(("Received extrainfo descriptor for bridge '%s', "
"but could not find bridge with that fingerprint.") %
router.fingerprint)
inserted = 0
logging.info("Inserting %d bridges into splitter..." % len(bridges))
for fingerprint, bridge in bridges.items():
# Skip insertion of bridges which are geolocated to be in one of the
# NO_DISTRIBUTION_COUNTRIES, a.k.a. the countries we don't distribute
# bridges from:
if bridge.country in state.NO_DISTRIBUTION_COUNTRIES:
logging.warn(
"Not distributing Bridge %s %s:%s in country %s!" %
(bridge, bridge.address, bridge.orPort, bridge.country))
else:
# If the bridge is not running, then it is skipped during the
# insertion process.
splitter.insert(bridge)
inserted += 1
logging.info("Done inserting %d bridges into splitter." % inserted)
if state.COLLECT_TIMESTAMPS:
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
state.save()
