def setUp(self):
self.bridges = copy.deepcopy(util.generateFakeBridges())
self.fd, self.fname = tempfile.mkstemp(suffix=".sqlite",
dir=os.getcwd())
bridgedb.Storage.initializeDBLock()
self.db = bridgedb.Storage.openDatabase(self.fname)
bridgedb.Storage.setDBFilename(self.fname)
key = 'fake-hmac-key'
self.splitter = Bridges.BridgeSplitter(key)
ringParams = Bridges.BridgeRingParameters(needPorts=[(443, 1)],
needFlags=[("Stable", 1)])
self.https_distributor = HTTPSDistributor(4,
crypto.getHMAC(
key,
"HTTPS-IP-Dist-Key"),
None,
answerParameters=ringParams)
self.moat_distributor = MoatDistributor(4,
crypto.getHMAC(
key, "Moat-Dist-Key"),
None,
answerParameters=ringParams)
self.unallocated_distributor = Bridges.UnallocatedHolder()
self.splitter.addRing(self.https_distributor.hashring, "https", p=10)
self.splitter.addRing(self.moat_distributor.hashring, "moat", p=10)
self.splitter.addRing(self.unallocated_distributor,
"unallocated",
p=10)
self.https_ring = self.splitter.ringsByName.get("https")
self.moat_ring = self.splitter.ringsByName.get("moat")
self.unallocated_ring = self.splitter.ringsByName.get("unallocated")
def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
countryblock = Bridges.CountryBlock()
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Clearing old blocked bridges")
countryblock.clear()
logging.info("Loading bridges")
status = {}
if hasattr(cfg, "STATUS_FILE"):
f = open(cfg.STATUS_FILE, 'r')
for ID, running, stable in Bridges.parseStatusFile(f):
status[ID] = running, stable
f.close()
if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
for fingerprint, countryCode in Bridges.parseCountryBlockFile(f):
countryblock.insert(fingerprint, countryCode)
f.close()
for fname in cfg.BRIDGE_FILES:
f = open(fname, 'r')
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
s = status.get(bridge.getID())
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
bridge.setBlockingCountries(
countryblock.getBlockingCountries(bridge.fingerprint))
splitter.insert(bridge)
f.close()
def setUp(self):
self.rings = [
Bridges.BridgeRing('fake-hmac-key-1'),
Bridges.BridgeRing('fake-hmac-key-2')
]
self.splitter = Bridges.FixedBridgeSplitter('fake-hmac-key',
self.rings)
def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
countryblock = Bridges.CountryBlock()
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Clearing old blocked bridges")
countryblock.clear()
logging.info("Loading bridges")
status = {}
if hasattr(cfg, "STATUS_FILE"):
f = open(cfg.STATUS_FILE, 'r')
for ID, running, stable in Bridges.parseStatusFile(f):
status[ID] = running, stable
f.close()
if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
for fingerprint, countryCode in Bridges.parseCountryBlockFile(f):
countryblock.insert(fingerprint, countryCode)
f.close()
for fname in cfg.BRIDGE_FILES:
f = open(fname, 'r')
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
s = status.get(bridge.getID())
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
bridge.setBlockingCountries(
countryblock.getBlockingCountries(bridge.fingerprint))
splitter.insert(bridge)
f.close()
def createBridgeRings(cfg, proxyList, key):
"""Create the bridge distributors defined by the config file
:type cfg: :class:`Conf`
:param cfg: The current configuration, including any in-memory settings
(i.e. settings whose values were not obtained from the config file,
but were set via a function somewhere)
:type proxyList: :class:`~bridgedb.proxy.ProxySet`
:param proxyList: The container for the IP addresses of any currently
known open proxies.
:param bytes key: Hashring master key
:rtype: tuple
:returns: A BridgeSplitter hashring, an
:class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an
:class:`~bridgedb.email.distributor.EmailDistributor` or None.
"""
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
hashring = Bridges.BridgeSplitter(crypto.getHMAC(key, "Hashring-Key"))
logging.debug("Created hashring: %r" % hashring)
# Create ring parameters.
ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS,
needFlags=cfg.FORCE_FLAGS)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
logging.debug("Setting up HTTPS Distributor...")
ipDistributor = HTTPSDistributor(cfg.N_IP_CLUSTERS,
crypto.getHMAC(
key, "HTTPS-IP-Dist-Key"),
proxyList,
answerParameters=ringParams)
hashring.addRing(ipDistributor.hashring, "https", cfg.HTTPS_SHARE)
# As appropriate, create an email-based distributor.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
logging.debug("Setting up Email Distributor...")
emailDistributor = EmailDistributor(
crypto.getHMAC(key, "Email-Dist-Key"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams,
whitelist=cfg.EMAIL_WHITELIST.copy())
hashring.addRing(emailDistributor.hashring, "email", cfg.EMAIL_SHARE)
# As appropriate, tell the hashring to leave some bridges unallocated.
if cfg.RESERVED_SHARE:
hashring.addRing(Bridges.UnallocatedHolder(), "unallocated",
cfg.RESERVED_SHARE)
# Add pseudo distributors to hashring
for pseudoRing in cfg.FILE_BUCKETS.keys():
hashring.addPseudoRing(pseudoRing)
return hashring, emailDistributor, ipDistributor
def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Loading bridges")
status = {}
if hasattr(cfg, "STATUS_FILE"):
f = open(cfg.STATUS_FILE, 'r')
for ID, running, stable in Bridges.parseStatusFile(f):
status[ID] = running, stable
for fname in cfg.BRIDGE_FILES:
f = open(fname, 'r')
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
s = status.get(bridge.getID())
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
splitter.insert(bridge)
f.close()
def loadProxyList(cfg):
ipset = {}
for fname in cfg.PROXY_LIST_FILES:
f = open(fname, "r")
for line in f:
line = line.strip()
if line.startswith("#"):
continue
elif Bridges.is_valid_ip(line):
ipset[line] = True
elif line:
logging.info("Skipping line %r in %s: not an IP.", line, fname)
f.close()
return ipset
def loadProxyList(cfg):
ipset = {}
for fname in cfg.PROXY_LIST_FILES:
f = open(fname, 'r')
for line in f:
line = line.strip()
if line.startswith("#"):
continue
elif Bridges.is_valid_ip(line):
ipset[line] = True
elif line:
logging.info("Skipping line %r in %s: not an IP.", line, fname)
f.close()
return ipset
def test_getDescriptorDigests(self):
sha1hash = hashlib.sha1()
ei_digest = 'abcdefghijklmno'
test = "this is a test line\nFollowed by another\n"
test += "extra-info-digest %s\n" % ei_digest
sha1hash.update(test)
digest = sha1hash.hexdigest()
test += "-----BEGIN SIGNATURE-----\n"
test += "This is a test line that should be skipped\n"
test += "-----END SIGNATURE-----\n"
digests = Bridges.getDescriptorDigests(StringIO(test))
self.failUnlessIn(digest, digests)
self.failUnlessEqual(ei_digest, digests[digest])
def test_getExtraInfoDigests(self):
sha1hash = hashlib.sha1()
test = "Many words and line all together\n"
test += "extra info is nothing like weather\n"
test += "it's certain to come, like the key in a hum\n"
test += "but sometimes without a transport and rum\n"
content = test
sha1hash.update(test)
digest = sha1hash.hexdigest()
test += "-----BEGIN SIGNATURE-----\n"
test += "But the rum can't save the world like you\n"
test += "-----END SIGNATURE-----\n"
digests = Bridges.getExtraInfoDigests(StringIO(test))
self.failUnlessIn(digest, digests)
self.failUnlessEqual(content, digests[digest].read())
def startup(cfg):
"""Parse bridges,
"""
# Expand any ~ characters in paths in the configuration.
cfg.BRIDGE_FILES = [ os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES ]
for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE",
"ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE",
"PIDFILE", "LOGFILE", "STATUS_FILE"):
v = getattr(cfg, key, None)
if v:
setattr(cfg, key, os.path.expanduser(v))
if hasattr(cfg, "PROXY_LIST_FILES"):
cfg.PROXY_LIST_FILES = [
os.path.expanduser(v) for v in cfg.PROXY_LIST_FILES ]
else:
cfg.PROXY_LIST_FILES = [ ]
# Write the pidfile.
if cfg.PIDFILE:
f = open(cfg.PIDFILE, 'w')
f.write("%s\n"%os.getpid())
f.close()
# Set up logging.
configureLogging(cfg)
#XXX import Server after logging is set up
# Otherwise, python will create a default handler that logs to
# the console and ignore further basicConfig calls
import bridgedb.Server as Server
# Load the master key, or create a new one.
key = getKey(cfg.MASTER_KEY_FILE)
# Initialize our DB file.
db = bridgedb.Storage.Database(cfg.DB_FILE+".sqlite",
cfg.DB_FILE)
bridgedb.Storage.setGlobalDB(db)
# Get a proxy list.
proxyList = ProxyCategory()
proxyList.replaceProxyList(loadProxyList(cfg))
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"))
# Create ring parameters.
forcePorts = getattr(cfg, "FORCE_PORTS")
forceFlags = getattr(cfg, "FORCE_FLAGS")
if not forcePorts: forcePorts = []
if not forceFlags: forceFlags = []
ringParams=Bridges.BridgeRingParameters(needPorts=forcePorts,
needFlags=forceFlags)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
categories = []
if proxyList.ipset:
categories.append(proxyList)
ipDistributor = Dist.IPBasedDistributor(
Dist.uniformMap,
cfg.N_IP_CLUSTERS,
Bridges.get_hmac(key, "HTTPS-IP-Dist-Key"),
categories,
answerParameters=ringParams)
splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
#webSchedule = Time.IntervalSchedule("day", 2)
webSchedule = Time.NoSchedule()
# As appropriate, create an email-based distributor.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
for d in cfg.EMAIL_DOMAINS:
cfg.EMAIL_DOMAIN_MAP[d] = d
emailDistributor = Dist.EmailBasedDistributor(
Bridges.get_hmac(key, "Email-Dist-Key"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams)
splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE)
#emailSchedule = Time.IntervalSchedule("day", 1)
emailSchedule = Time.NoSchedule()
# As appropriate, tell the splitter to leave some bridges unallocated.
if cfg.RESERVED_SHARE:
splitter.addRing(Bridges.UnallocatedHolder(),
"unallocated",
cfg.RESERVED_SHARE)
# Add pseudo distributors to splitter
for p in cfg.FILE_BUCKETS.keys():
splitter.addPseudoRing(p)
# Make the parse-bridges function get re-called on SIGHUP.
def reload():
logging.info("Caught SIGHUP")
load(cfg, splitter, clear=True)
proxyList.replaceProxyList(loadProxyList(cfg))
logging.info("%d bridges loaded", len(splitter))
if emailDistributor:
logging.info("%d for email", len(emailDistributor.ring))
if ipDistributor:
logging.info("%d for web:", len(ipDistributor.splitter))
logging.info(" by location set: %s",
" ".join(str(len(r)) for r in ipDistributor.rings))
logging.info(" by category set: %s",
" ".join(str(len(r)) for r in ipDistributor.categoryRings))
logging.info("Here are all known bridges in the category section:")
for r in ipDistributor.categoryRings:
for name, b in r.bridges.items():
logging.info("%s" % b.getConfigLine(True))
# Dump bridge pool assignments to disk.
try:
f = open(cfg.ASSIGNMENTS_FILE, 'a')
f.write("bridge-pool-assignment %s\n" %
time.strftime("%Y-%m-%d %H:%M:%S"))
splitter.dumpAssignments(f)
f.close()
except IOError:
logging.info("I/O error while writing assignments")
global _reloadFn
_reloadFn = reload
signal.signal(signal.SIGHUP, _handleSIGHUP)
# And actually load it to start.
reload()
# Configure HTTP and/or HTTPS servers.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
Server.addWebServer(cfg, ipDistributor, webSchedule)
# Configure Email servers.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
Server.addSMTPServer(cfg, emailDistributor, emailSchedule)
# Actually run the servers.
try:
logging.info("Starting reactors.")
Server.runServers()
finally:
db.close()
if cfg.PIDFILE:
os.unlink(cfg.PIDFILE)
def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Loading bridges")
status = {}
addresses = {}
timestamps = {}
if hasattr(cfg, "STATUS_FILE"):
logging.info("Opening Network Status document %s", os.path.abspath(cfg.STATUS_FILE))
f = open(cfg.STATUS_FILE, "r")
for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f):
status[ID] = running, stable
addresses[ID] = or_addresses
if ID in timestamps.keys():
timestamps[ID].append(timestamp)
else:
timestamps[ID] = [timestamp]
# transports[ID] = transports
logging.debug("Closing status document")
f.close()
bridges = {}
db = bridgedb.Storage.getDB()
for fname in cfg.BRIDGE_FILES:
logging.info("Opening cached Descriptor document %s", fname)
logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE)
f = open(fname, "r")
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
if bridge.getID() in bridges:
logging.warn("Parsed bridge that we've already added. Skipping.")
logging.debug(" Bridge: %s" % bridge.getID())
continue
else:
bridges[bridge.getID()] = bridge
s = status.get(bridge.getID())
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
bridge.or_addresses = addresses.get(bridge.getID())
splitter.insert(bridge)
# add or update BridgeHistory entries into the database
# XXX: what do we do with all these or_addresses?
# The bridge stability metrics are only concerned with a single ip:port
# So for now, we will only consider the bridges primary IP:port
if bridge.getID() in timestamps.keys():
ts = timestamps[bridge.getID()][:]
ts.sort()
for timestamp in ts:
bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp)
logging.debug("Closing descriptor document")
f.close()
# read pluggable transports from extra-info document
# XXX: should read from networkstatus after bridge-authority
# does a reachability test
if hasattr(cfg, "EXTRA_INFO_FILE"):
logging.info("Opening Extra Info document %s", os.path.abspath(cfg.EXTRA_INFO_FILE))
f = open(cfg.EXTRA_INFO_FILE, "r")
for transport in Bridges.parseExtraInfoFile(f):
ID, method_name, address, port, argdict = transport
if bridges[ID].running:
logging.debug(" Appending transport to running bridge")
bridges[ID].transports.append(
Bridges.PluggableTransport(bridges[ID], method_name, address, port, argdict)
)
assert bridges[ID].transports, "We added a transport but it disappeared!"
logging.debug("Closing extra-info document")
f.close()
if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
logging.info("Opening Blocking Countries file %s", os.path.abspath(cfg.COUNTRY_BLOCK_FILE))
f = open(cfg.COUNTRY_BLOCK_FILE, "r")
for ID, address, portlist, countries in Bridges.parseCountryBlockFile(f):
if ID in bridges.keys() and bridges[ID].running:
for port in portlist:
logging.debug(":.( Tears! %s blocked %s %s:%s", countries, bridges[ID].fingerprint, address, port)
try:
bridges[ID].blockingCountries["%s:%s" % (address, port)].update(countries)
except KeyError:
bridges[ID].blockingCountries["%s:%s" % (address, port)] = set(countries)
logging.debug("Closing blocking-countries document")
f.close()
bridges = None
def setUp(self):
self.ring = Bridges.BridgeRing('fake-hmac-key')
def startup(cfg):
"""Parse bridges,
"""
# Expand any ~ characters in paths in the configuration.
cfg.BRIDGE_FILES = [os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES]
for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE",
"ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE",
"PIDFILE", "LOGFILE", "STATUS_FILE"):
v = getattr(cfg, key, None)
if v:
setattr(cfg, key, os.path.expanduser(v))
if hasattr(cfg, "PROXY_LIST_FILES"):
cfg.PROXY_LIST_FILES = [
os.path.expanduser(v) for v in cfg.PROXY_LIST_FILES
]
else:
cfg.PROXY_LIST_FILES = []
# Write the pidfile.
if cfg.PIDFILE:
f = open(cfg.PIDFILE, 'w')
f.write("%s\n" % os.getpid())
f.close()
# Set up logging.
configureLogging(cfg)
#XXX import Server after logging is set up
# Otherwise, python will create a default handler that logs to
# the console and ignore further basicConfig calls
import bridgedb.Server as Server
# Load the master key, or create a new one.
key = getKey(cfg.MASTER_KEY_FILE)
# Initialize our DB file.
db = bridgedb.Storage.Database(cfg.DB_FILE + ".sqlite", cfg.DB_FILE)
bridgedb.Storage.setGlobalDB(db)
# Get a proxy list.
proxyList = ProxyCategory()
proxyList.replaceProxyList(loadProxyList(cfg))
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"))
# Create ring parameters.
forcePorts = getattr(cfg, "FORCE_PORTS")
forceFlags = getattr(cfg, "FORCE_FLAGS")
if not forcePorts: forcePorts = []
if not forceFlags: forceFlags = []
ringParams = Bridges.BridgeRingParameters(needPorts=forcePorts,
needFlags=forceFlags)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
categories = []
if proxyList.ipset:
categories.append(proxyList)
ipDistributor = Dist.IPBasedDistributor(Dist.uniformMap,
cfg.N_IP_CLUSTERS,
Bridges.get_hmac(
key, "HTTPS-IP-Dist-Key"),
categories,
answerParameters=ringParams)
splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
#webSchedule = Time.IntervalSchedule("day", 2)
webSchedule = Time.NoSchedule()
# As appropriate, create an email-based distributor.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
for d in cfg.EMAIL_DOMAINS:
cfg.EMAIL_DOMAIN_MAP[d] = d
emailDistributor = Dist.EmailBasedDistributor(
Bridges.get_hmac(key, "Email-Dist-Key"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams)
splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE)
#emailSchedule = Time.IntervalSchedule("day", 1)
emailSchedule = Time.NoSchedule()
# As appropriate, tell the splitter to leave some bridges unallocated.
if cfg.RESERVED_SHARE:
splitter.addRing(Bridges.UnallocatedHolder(), "unallocated",
cfg.RESERVED_SHARE)
# Add pseudo distributors to splitter
for p in cfg.FILE_BUCKETS.keys():
splitter.addPseudoRing(p)
# Make the parse-bridges function get re-called on SIGHUP.
def reload():
logging.info("Caught SIGHUP")
load(cfg, splitter, clear=True)
proxyList.replaceProxyList(loadProxyList(cfg))
logging.info("%d bridges loaded", len(splitter))
if emailDistributor:
logging.info("%d for email", len(emailDistributor.ring))
if ipDistributor:
logging.info("%d for web:", len(ipDistributor.splitter))
logging.info(" by location set: %s",
" ".join(str(len(r)) for r in ipDistributor.rings))
logging.info(
" by category set: %s",
" ".join(str(len(r)) for r in ipDistributor.categoryRings))
logging.info("Here are all known bridges in the category section:")
for r in ipDistributor.categoryRings:
for name, b in r.bridges.items():
logging.info("%s" % b.getConfigLine(True))
# Dump bridge pool assignments to disk.
try:
f = open(cfg.ASSIGNMENTS_FILE, 'a')
f.write("bridge-pool-assignment %s\n" %
time.strftime("%Y-%m-%d %H:%M:%S"))
splitter.dumpAssignments(f)
f.close()
except IOError:
logging.info("I/O error while writing assignments")
global _reloadFn
_reloadFn = reload
signal.signal(signal.SIGHUP, _handleSIGHUP)
# And actually load it to start.
reload()
# Configure HTTP and/or HTTPS servers.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
Server.addWebServer(cfg, ipDistributor, webSchedule)
# Configure Email servers.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
Server.addSMTPServer(cfg, emailDistributor, emailSchedule)
# Actually run the servers.
try:
logging.info("Starting reactors.")
Server.runServers()
finally:
db.close()
if cfg.PIDFILE:
os.unlink(cfg.PIDFILE)
def load(state, splitter, clear=False):
"""Read and parse all descriptors, and load into a bridge splitter.
Read all the appropriate bridge files from the saved
:class:`~bridgedb.persistent.State`, parse and validate them, and then
store them into our ``state.splitter`` instance. The ``state`` will be
saved again at the end of this function.
:type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>`
:param splitter: A class which provides a mechanism for HMACing
Bridges in order to assign them to hashrings.
:param boolean clear: If True, clear all previous bridges from the
splitter before parsing for new ones.
"""
if not state:
logging.fatal("bridgedb.Main.load() could not retrieve state!")
sys.exit(2)
if clear:
logging.info("Clearing old bridges...")
splitter.clear()
logging.info("Loading bridges...")
bridges = {}
status = {}
addresses = {}
timestamps = {}
bridges = {}
desc_digests = {}
ei_digests = {}
logging.info("Opening network status file: %s" % state.STATUS_FILE)
f = open(state.STATUS_FILE, 'r')
for (ID, nickname, desc_digest, running, stable,
ORaddr, ORport, or_addresses,
timestamp) in Bridges.parseStatusFile(f):
bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID,
or_addresses=or_addresses)
bridge.assertOK()
bridge.setStatus(running, stable)
bridge.setDescriptorDigest(desc_digest)
bridges[ID] = bridge
if ID in timestamps.keys():
timestamps[ID].append(timestamp)
else:
timestamps[ID] = [timestamp]
logging.debug("Closing network status file")
f.close()
for fname in state.BRIDGE_FILES:
logging.info("Opening bridge-server-descriptor file: '%s'" % fname)
f = open(fname, 'r')
desc_digests.update(Bridges.getDescriptorDigests(f))
if state.COLLECT_TIMESTAMPS:
for bridge in bridges.values():
if bridge.getID() in timestamps.keys():
ts = timestamps[bridge.getID()][:]
ts.sort()
for timestamp in ts:
logging.debug(
"Adding/updating timestamps in BridgeHistory for "\
"'%s' in database: %s"
% (bridge.fingerprint, timestamp))
bridgedb.Stability.addOrUpdateBridgeHistory(
bridge, timestamp)
logging.debug("Closing bridge-server-descriptor file: '%s'" % fname)
f.close()
for ID in bridges.keys():
bridge = bridges[ID]
if bridge.desc_digest in desc_digests:
bridge.setVerified()
bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest])
# We attempt to insert all bridges. If the bridge is not
# running, then it is skipped during the insertion process.
splitter.insert(bridge)
# read pluggable transports from extra-info document
# XXX: should read from networkstatus after bridge-authority
# does a reachability test
for filename in state.EXTRA_INFO_FILES:
logging.info("Opening extra-info file: '%s'" % filename)
f = open(filename, 'r')
for transport in Bridges.parseExtraInfoFile(f):
ID, method_name, address, port, argdict = transport
try:
if bridges[ID].running:
logging.info("Adding %s transport to running bridge"
% method_name)
bridgePT = Bridges.PluggableTransport(
bridges[ID], method_name, address, port, argdict)
bridges[ID].transports.append(bridgePT)
if not bridgePT in bridges[ID].transports:
logging.critical(
"Added a transport, but it disappeared!",
"\tTransport: %r" % bridgePT)
except KeyError as error:
logging.error("Could not find bridge with fingerprint '%s'."
% Bridges.toHex(ID))
logging.debug("Closing extra-info file: '%s'" % filename)
f.close()
if state.COUNTRY_BLOCK_FILE:
logging.info("Opening Blocking Countries file %s"
% state.COUNTRY_BLOCK_FILE)
f = open(state.COUNTRY_BLOCK_FILE)
# Identity digest, primary OR address, portlist, country codes
for ID, addr, portlist, cc in Bridges.parseCountryBlockFile(f):
if ID in bridges.keys() and bridges[ID].running:
for port in portlist:
addrport = "{0}:{1}".format(addr, port)
logging.debug(":'( Tears! %s blocked bridge %s at %s"
% (cc, bridges[ID].fingerprint, addrport))
try:
bridges[ID].blockingCountries[addrport].update(cc)
except KeyError:
bridges[ID].blockingCountries[addrport] = set(cc)
logging.debug("Closing blocking-countries document")
f.close()
def updateBridgeHistory(bridges, timestamps):
if not hasattr(state, 'config'):
logging.info("updateBridgeHistory(): Config file not set "\
"in State file.")
return
if state.COLLECT_TIMESTAMPS:
logging.debug("Beginning bridge stability calculations")
for bridge in bridges.values():
if bridge.getID() in timestamps.keys():
ts = timestamps[bridge.getID()][:]
ts.sort()
for timestamp in ts:
logging.debug(
"Updating BridgeHistory timestamps for %s: %s"
% (bridge.fingerprint, timestamp))
bridgedb.Stability.addOrUpdateBridgeHistory(
bridge, timestamp)
logging.debug("Stability calculations complete")
reactor.callInThread(updateBridgeHistory, bridges, timestamps)
bridges = None
state.save()
return
def load(cfg, splitter, clear=False):
"""Read all the bridge files from cfg, and pass them into a splitter
object.
"""
if clear:
logging.info("Clearing old bridges")
splitter.clear()
logging.info("Loading bridges")
status = {}
addresses = {}
timestamps = {}
if hasattr(cfg, "STATUS_FILE"):
logging.info("Opening Network Status document %s",
os.path.abspath(cfg.STATUS_FILE))
f = open(cfg.STATUS_FILE, 'r')
for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(
f):
status[ID] = running, stable
addresses[ID] = or_addresses
if ID in timestamps.keys(): timestamps[ID].append(timestamp)
else: timestamps[ID] = [timestamp]
#transports[ID] = transports
logging.debug("Closing status document")
f.close()
bridges = {}
db = bridgedb.Storage.getDB()
for fname in cfg.BRIDGE_FILES:
logging.info("Opening cached Descriptor document %s", fname)
logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE)
f = open(fname, 'r')
for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
if bridge.getID() in bridges:
logging.warn(
"Parsed bridge that we've already added. Skipping.")
logging.debug(" Bridge: %s" % bridge.getID())
continue
else:
bridges[bridge.getID()] = bridge
s = status.get(bridge.getID())
if s is not None:
running, stable = s
bridge.setStatus(running=running, stable=stable)
bridge.or_addresses = addresses.get(bridge.getID())
splitter.insert(bridge)
# add or update BridgeHistory entries into the database
# XXX: what do we do with all these or_addresses?
# The bridge stability metrics are only concerned with a single ip:port
# So for now, we will only consider the bridges primary IP:port
if bridge.getID() in timestamps.keys():
ts = timestamps[bridge.getID()][:]
ts.sort()
for timestamp in ts:
bridgedb.Stability.addOrUpdateBridgeHistory(
bridge, timestamp)
logging.debug("Closing descriptor document")
f.close()
# read pluggable transports from extra-info document
# XXX: should read from networkstatus after bridge-authority
# does a reachability test
if hasattr(cfg, "EXTRA_INFO_FILE"):
logging.info("Opening Extra Info document %s",
os.path.abspath(cfg.EXTRA_INFO_FILE))
f = open(cfg.EXTRA_INFO_FILE, 'r')
for transport in Bridges.parseExtraInfoFile(f):
ID, method_name, address, port, argdict = transport
if bridges[ID].running:
logging.debug(" Appending transport to running bridge")
bridges[ID].transports.append(
Bridges.PluggableTransport(bridges[ID], method_name,
address, port, argdict))
assert bridges[
ID].transports, "We added a transport but it disappeared!"
logging.debug("Closing extra-info document")
f.close()
if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
logging.info("Opening Blocking Countries file %s",
os.path.abspath(cfg.COUNTRY_BLOCK_FILE))
f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
for ID, address, portlist, countries in Bridges.parseCountryBlockFile(
f):
if ID in bridges.keys() and bridges[ID].running:
for port in portlist:
logging.debug(":.( Tears! %s blocked %s %s:%s", countries,
bridges[ID].fingerprint, address, port)
try:
bridges[ID].blockingCountries["%s:%s" % \
(address, port)].update(countries)
except KeyError:
bridges[ID].blockingCountries["%s:%s" % \
(address, port)] = set(countries)
logging.debug("Closing blocking-countries document")
f.close()
bridges = None