def setUp(self):
self.bridges = copy.deepcopy(util.generateFakeBridges())
self.fd, self.fname = tempfile.mkstemp(suffix=".sqlite",
dir=os.getcwd())
bridgedb.Storage.initializeDBLock()
self.db = bridgedb.Storage.openDatabase(self.fname)
bridgedb.Storage.setDBFilename(self.fname)
key = 'fake-hmac-key'
self.splitter = Bridges.BridgeSplitter(key)
ringParams = Bridges.BridgeRingParameters(needPorts=[(443, 1)],
needFlags=[("Stable", 1)])
self.https_distributor = HTTPSDistributor(4,
crypto.getHMAC(
key,
"HTTPS-IP-Dist-Key"),
None,
answerParameters=ringParams)
self.moat_distributor = MoatDistributor(4,
crypto.getHMAC(
key, "Moat-Dist-Key"),
None,
answerParameters=ringParams)
self.unallocated_distributor = Bridges.UnallocatedHolder()
self.splitter.addRing(self.https_distributor.hashring, "https", p=10)
self.splitter.addRing(self.moat_distributor.hashring, "moat", p=10)
self.splitter.addRing(self.unallocated_distributor,
"unallocated",
p=10)
self.https_ring = self.splitter.ringsByName.get("https")
self.moat_ring = self.splitter.ringsByName.get("moat")
self.unallocated_ring = self.splitter.ringsByName.get("unallocated")
def createBridgeRings(cfg, proxyList, key):
"""Create the bridge distributors defined by the config file
:type cfg: :class:`Conf`
:param cfg: The current configuration, including any in-memory settings
(i.e. settings whose values were not obtained from the config file,
but were set via a function somewhere)
:type proxyList: :class:`~bridgedb.proxy.ProxySet`
:param proxyList: The container for the IP addresses of any currently
known open proxies.
:param bytes key: Hashring master key
:rtype: tuple
:returns: A BridgeSplitter hashring, an
:class:`~bridgedb.https.distributor.HTTPSDistributor` or None, and an
:class:`~bridgedb.email.distributor.EmailDistributor` or None.
"""
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
hashring = Bridges.BridgeSplitter(crypto.getHMAC(key, "Hashring-Key"))
logging.debug("Created hashring: %r" % hashring)
# Create ring parameters.
ringParams = Bridges.BridgeRingParameters(needPorts=cfg.FORCE_PORTS,
needFlags=cfg.FORCE_FLAGS)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
logging.debug("Setting up HTTPS Distributor...")
ipDistributor = HTTPSDistributor(cfg.N_IP_CLUSTERS,
crypto.getHMAC(
key, "HTTPS-IP-Dist-Key"),
proxyList,
answerParameters=ringParams)
hashring.addRing(ipDistributor.hashring, "https", cfg.HTTPS_SHARE)
# As appropriate, create an email-based distributor.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
logging.debug("Setting up Email Distributor...")
emailDistributor = EmailDistributor(
crypto.getHMAC(key, "Email-Dist-Key"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams,
whitelist=cfg.EMAIL_WHITELIST.copy())
hashring.addRing(emailDistributor.hashring, "email", cfg.EMAIL_SHARE)
# As appropriate, tell the hashring to leave some bridges unallocated.
if cfg.RESERVED_SHARE:
hashring.addRing(Bridges.UnallocatedHolder(), "unallocated",
cfg.RESERVED_SHARE)
# Add pseudo distributors to hashring
for pseudoRing in cfg.FILE_BUCKETS.keys():
hashring.addPseudoRing(pseudoRing)
return hashring, emailDistributor, ipDistributor
def startup(cfg):
"""Parse bridges,
"""
# Expand any ~ characters in paths in the configuration.
cfg.BRIDGE_FILES = [os.path.expanduser(fn) for fn in cfg.BRIDGE_FILES]
for key in ("RUN_IN_DIR", "DB_FILE", "DB_LOG_FILE", "MASTER_KEY_FILE",
"ASSIGNMENTS_FILE", "HTTPS_CERT_FILE", "HTTPS_KEY_FILE",
"PIDFILE", "LOGFILE", "STATUS_FILE"):
v = getattr(cfg, key, None)
if v:
setattr(cfg, key, os.path.expanduser(v))
if hasattr(cfg, "PROXY_LIST_FILES"):
cfg.PROXY_LIST_FILES = [
os.path.expanduser(v) for v in cfg.PROXY_LIST_FILES
]
else:
cfg.PROXY_LIST_FILES = []
# Write the pidfile.
if cfg.PIDFILE:
f = open(cfg.PIDFILE, 'w')
f.write("%s\n" % os.getpid())
f.close()
# Set up logging.
configureLogging(cfg)
#XXX import Server after logging is set up
# Otherwise, python will create a default handler that logs to
# the console and ignore further basicConfig calls
import bridgedb.Server as Server
# Load the master key, or create a new one.
key = getKey(cfg.MASTER_KEY_FILE)
# Initialize our DB file.
db = bridgedb.Storage.Database(cfg.DB_FILE + ".sqlite", cfg.DB_FILE)
bridgedb.Storage.setGlobalDB(db)
# Get a proxy list.
proxyList = ProxyCategory()
proxyList.replaceProxyList(loadProxyList(cfg))
# Create a BridgeSplitter to assign the bridges to the different
# distributors.
splitter = Bridges.BridgeSplitter(Bridges.get_hmac(key, "Splitter-Key"))
# Create ring parameters.
forcePorts = getattr(cfg, "FORCE_PORTS")
forceFlags = getattr(cfg, "FORCE_FLAGS")
if not forcePorts: forcePorts = []
if not forceFlags: forceFlags = []
ringParams = Bridges.BridgeRingParameters(needPorts=forcePorts,
needFlags=forceFlags)
emailDistributor = ipDistributor = None
# As appropriate, create an IP-based distributor.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
categories = []
if proxyList.ipset:
categories.append(proxyList)
ipDistributor = Dist.IPBasedDistributor(Dist.uniformMap,
cfg.N_IP_CLUSTERS,
Bridges.get_hmac(
key, "HTTPS-IP-Dist-Key"),
categories,
answerParameters=ringParams)
splitter.addRing(ipDistributor, "https", cfg.HTTPS_SHARE)
#webSchedule = Time.IntervalSchedule("day", 2)
webSchedule = Time.NoSchedule()
# As appropriate, create an email-based distributor.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
for d in cfg.EMAIL_DOMAINS:
cfg.EMAIL_DOMAIN_MAP[d] = d
emailDistributor = Dist.EmailBasedDistributor(
Bridges.get_hmac(key, "Email-Dist-Key"),
cfg.EMAIL_DOMAIN_MAP.copy(),
cfg.EMAIL_DOMAIN_RULES.copy(),
answerParameters=ringParams)
splitter.addRing(emailDistributor, "email", cfg.EMAIL_SHARE)
#emailSchedule = Time.IntervalSchedule("day", 1)
emailSchedule = Time.NoSchedule()
# As appropriate, tell the splitter to leave some bridges unallocated.
if cfg.RESERVED_SHARE:
splitter.addRing(Bridges.UnallocatedHolder(), "unallocated",
cfg.RESERVED_SHARE)
# Add pseudo distributors to splitter
for p in cfg.FILE_BUCKETS.keys():
splitter.addPseudoRing(p)
# Make the parse-bridges function get re-called on SIGHUP.
def reload():
logging.info("Caught SIGHUP")
load(cfg, splitter, clear=True)
proxyList.replaceProxyList(loadProxyList(cfg))
logging.info("%d bridges loaded", len(splitter))
if emailDistributor:
logging.info("%d for email", len(emailDistributor.ring))
if ipDistributor:
logging.info("%d for web:", len(ipDistributor.splitter))
logging.info(" by location set: %s",
" ".join(str(len(r)) for r in ipDistributor.rings))
logging.info(
" by category set: %s",
" ".join(str(len(r)) for r in ipDistributor.categoryRings))
logging.info("Here are all known bridges in the category section:")
for r in ipDistributor.categoryRings:
for name, b in r.bridges.items():
logging.info("%s" % b.getConfigLine(True))
# Dump bridge pool assignments to disk.
try:
f = open(cfg.ASSIGNMENTS_FILE, 'a')
f.write("bridge-pool-assignment %s\n" %
time.strftime("%Y-%m-%d %H:%M:%S"))
splitter.dumpAssignments(f)
f.close()
except IOError:
logging.info("I/O error while writing assignments")
global _reloadFn
_reloadFn = reload
signal.signal(signal.SIGHUP, _handleSIGHUP)
# And actually load it to start.
reload()
# Configure HTTP and/or HTTPS servers.
if cfg.HTTPS_DIST and cfg.HTTPS_SHARE:
Server.addWebServer(cfg, ipDistributor, webSchedule)
# Configure Email servers.
if cfg.EMAIL_DIST and cfg.EMAIL_SHARE:
Server.addSMTPServer(cfg, emailDistributor, emailSchedule)
# Actually run the servers.
try:
logging.info("Starting reactors.")
Server.runServers()
finally:
db.close()
if cfg.PIDFILE:
os.unlink(cfg.PIDFILE)
