def load(cfg, splitter, clear=False): """Read all the bridge files from cfg, and pass them into a splitter object. """ countryblock = Bridges.CountryBlock() if clear: logging.info("Clearing old bridges") splitter.clear() logging.info("Clearing old blocked bridges") countryblock.clear() logging.info("Loading bridges") status = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') for ID, running, stable in Bridges.parseStatusFile(f): status[ID] = running, stable f.close() if hasattr(cfg, "COUNTRY_BLOCK_FILE"): f = open(cfg.COUNTRY_BLOCK_FILE, 'r') for fingerprint, countryCode in Bridges.parseCountryBlockFile(f): countryblock.insert(fingerprint, countryCode) f.close() for fname in cfg.BRIDGE_FILES: f = open(fname, 'r') for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): s = status.get(bridge.getID()) if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) bridge.setBlockingCountries( countryblock.getBlockingCountries(bridge.fingerprint)) splitter.insert(bridge) f.close()
def load(cfg, splitter, clear=False): """Read all the bridge files from cfg, and pass them into a splitter object. """ if clear: logging.info("Clearing old bridges") splitter.clear() logging.info("Loading bridges") status = {} if hasattr(cfg, "STATUS_FILE"): f = open(cfg.STATUS_FILE, 'r') for ID, running, stable in Bridges.parseStatusFile(f): status[ID] = running, stable for fname in cfg.BRIDGE_FILES: f = open(fname, 'r') for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): s = status.get(bridge.getID()) if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) splitter.insert(bridge) f.close()
def load(cfg, splitter, clear=False): """Read all the bridge files from cfg, and pass them into a splitter object. """ if clear: logging.info("Clearing old bridges") splitter.clear() logging.info("Loading bridges") status = {} addresses = {} timestamps = {} if hasattr(cfg, "STATUS_FILE"): logging.info("Opening Network Status document %s", os.path.abspath(cfg.STATUS_FILE)) f = open(cfg.STATUS_FILE, "r") for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f): status[ID] = running, stable addresses[ID] = or_addresses if ID in timestamps.keys(): timestamps[ID].append(timestamp) else: timestamps[ID] = [timestamp] # transports[ID] = transports logging.debug("Closing status document") f.close() bridges = {} db = bridgedb.Storage.getDB() for fname in cfg.BRIDGE_FILES: logging.info("Opening cached Descriptor document %s", fname) logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE) f = open(fname, "r") for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): if bridge.getID() in bridges: logging.warn("Parsed bridge that we've already added. Skipping.") logging.debug(" Bridge: %s" % bridge.getID()) continue else: bridges[bridge.getID()] = bridge s = status.get(bridge.getID()) if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) bridge.or_addresses = addresses.get(bridge.getID()) splitter.insert(bridge) # add or update BridgeHistory entries into the database # XXX: what do we do with all these or_addresses? # The bridge stability metrics are only concerned with a single ip:port # So for now, we will only consider the bridges primary IP:port if bridge.getID() in timestamps.keys(): ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp) logging.debug("Closing descriptor document") f.close() # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority # does a reachability test if hasattr(cfg, "EXTRA_INFO_FILE"): logging.info("Opening Extra Info document %s", os.path.abspath(cfg.EXTRA_INFO_FILE)) f = open(cfg.EXTRA_INFO_FILE, "r") for transport in Bridges.parseExtraInfoFile(f): ID, method_name, address, port, argdict = transport if bridges[ID].running: logging.debug(" Appending transport to running bridge") bridges[ID].transports.append( Bridges.PluggableTransport(bridges[ID], method_name, address, port, argdict) ) assert bridges[ID].transports, "We added a transport but it disappeared!" logging.debug("Closing extra-info document") f.close() if hasattr(cfg, "COUNTRY_BLOCK_FILE"): logging.info("Opening Blocking Countries file %s", os.path.abspath(cfg.COUNTRY_BLOCK_FILE)) f = open(cfg.COUNTRY_BLOCK_FILE, "r") for ID, address, portlist, countries in Bridges.parseCountryBlockFile(f): if ID in bridges.keys() and bridges[ID].running: for port in portlist: logging.debug(":.( Tears! %s blocked %s %s:%s", countries, bridges[ID].fingerprint, address, port) try: bridges[ID].blockingCountries["%s:%s" % (address, port)].update(countries) except KeyError: bridges[ID].blockingCountries["%s:%s" % (address, port)] = set(countries) logging.debug("Closing blocking-countries document") f.close() bridges = None
def load(state, splitter, clear=False): """Read and parse all descriptors, and load into a bridge splitter. Read all the appropriate bridge files from the saved :class:`~bridgedb.persistent.State`, parse and validate them, and then store them into our ``state.splitter`` instance. The ``state`` will be saved again at the end of this function. :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>` :param splitter: A class which provides a mechanism for HMACing Bridges in order to assign them to hashrings. :param boolean clear: If True, clear all previous bridges from the splitter before parsing for new ones. """ if not state: logging.fatal("bridgedb.Main.load() could not retrieve state!") sys.exit(2) if clear: logging.info("Clearing old bridges...") splitter.clear() logging.info("Loading bridges...") bridges = {} status = {} addresses = {} timestamps = {} bridges = {} desc_digests = {} ei_digests = {} logging.info("Opening network status file: %s" % state.STATUS_FILE) f = open(state.STATUS_FILE, 'r') for (ID, nickname, desc_digest, running, stable, ORaddr, ORport, or_addresses, timestamp) in Bridges.parseStatusFile(f): bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID, or_addresses=or_addresses) bridge.assertOK() bridge.setStatus(running, stable) bridge.setDescriptorDigest(desc_digest) bridges[ID] = bridge if ID in timestamps.keys(): timestamps[ID].append(timestamp) else: timestamps[ID] = [timestamp] logging.debug("Closing network status file") f.close() for fname in state.BRIDGE_FILES: logging.info("Opening bridge-server-descriptor file: '%s'" % fname) f = open(fname, 'r') desc_digests.update(Bridges.getDescriptorDigests(f)) if state.COLLECT_TIMESTAMPS: for bridge in bridges.values(): if bridge.getID() in timestamps.keys(): ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: logging.debug( "Adding/updating timestamps in BridgeHistory for "\ "'%s' in database: %s" % (bridge.fingerprint, timestamp)) bridgedb.Stability.addOrUpdateBridgeHistory( bridge, timestamp) logging.debug("Closing bridge-server-descriptor file: '%s'" % fname) f.close() for ID in bridges.keys(): bridge = bridges[ID] if bridge.desc_digest in desc_digests: bridge.setVerified() bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest]) # We attempt to insert all bridges. If the bridge is not # running, then it is skipped during the insertion process. splitter.insert(bridge) # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority # does a reachability test for filename in state.EXTRA_INFO_FILES: logging.info("Opening extra-info file: '%s'" % filename) f = open(filename, 'r') for transport in Bridges.parseExtraInfoFile(f): ID, method_name, address, port, argdict = transport try: if bridges[ID].running: logging.info("Adding %s transport to running bridge" % method_name) bridgePT = Bridges.PluggableTransport( bridges[ID], method_name, address, port, argdict) bridges[ID].transports.append(bridgePT) if not bridgePT in bridges[ID].transports: logging.critical( "Added a transport, but it disappeared!", "\tTransport: %r" % bridgePT) except KeyError as error: logging.error("Could not find bridge with fingerprint '%s'." % Bridges.toHex(ID)) logging.debug("Closing extra-info file: '%s'" % filename) f.close() if state.COUNTRY_BLOCK_FILE: logging.info("Opening Blocking Countries file %s" % state.COUNTRY_BLOCK_FILE) f = open(state.COUNTRY_BLOCK_FILE) # Identity digest, primary OR address, portlist, country codes for ID, addr, portlist, cc in Bridges.parseCountryBlockFile(f): if ID in bridges.keys() and bridges[ID].running: for port in portlist: addrport = "{0}:{1}".format(addr, port) logging.debug(":'( Tears! %s blocked bridge %s at %s" % (cc, bridges[ID].fingerprint, addrport)) try: bridges[ID].blockingCountries[addrport].update(cc) except KeyError: bridges[ID].blockingCountries[addrport] = set(cc) logging.debug("Closing blocking-countries document") f.close() def updateBridgeHistory(bridges, timestamps): if not hasattr(state, 'config'): logging.info("updateBridgeHistory(): Config file not set "\ "in State file.") return if state.COLLECT_TIMESTAMPS: logging.debug("Beginning bridge stability calculations") for bridge in bridges.values(): if bridge.getID() in timestamps.keys(): ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: logging.debug( "Updating BridgeHistory timestamps for %s: %s" % (bridge.fingerprint, timestamp)) bridgedb.Stability.addOrUpdateBridgeHistory( bridge, timestamp) logging.debug("Stability calculations complete") reactor.callInThread(updateBridgeHistory, bridges, timestamps) bridges = None state.save() return
def load(cfg, splitter, clear=False): """Read all the bridge files from cfg, and pass them into a splitter object. """ if clear: logging.info("Clearing old bridges") splitter.clear() logging.info("Loading bridges") status = {} addresses = {} timestamps = {} if hasattr(cfg, "STATUS_FILE"): logging.info("Opening Network Status document %s", os.path.abspath(cfg.STATUS_FILE)) f = open(cfg.STATUS_FILE, 'r') for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile( f): status[ID] = running, stable addresses[ID] = or_addresses if ID in timestamps.keys(): timestamps[ID].append(timestamp) else: timestamps[ID] = [timestamp] #transports[ID] = transports logging.debug("Closing status document") f.close() bridges = {} db = bridgedb.Storage.getDB() for fname in cfg.BRIDGE_FILES: logging.info("Opening cached Descriptor document %s", fname) logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE) f = open(fname, 'r') for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE): if bridge.getID() in bridges: logging.warn( "Parsed bridge that we've already added. Skipping.") logging.debug(" Bridge: %s" % bridge.getID()) continue else: bridges[bridge.getID()] = bridge s = status.get(bridge.getID()) if s is not None: running, stable = s bridge.setStatus(running=running, stable=stable) bridge.or_addresses = addresses.get(bridge.getID()) splitter.insert(bridge) # add or update BridgeHistory entries into the database # XXX: what do we do with all these or_addresses? # The bridge stability metrics are only concerned with a single ip:port # So for now, we will only consider the bridges primary IP:port if bridge.getID() in timestamps.keys(): ts = timestamps[bridge.getID()][:] ts.sort() for timestamp in ts: bridgedb.Stability.addOrUpdateBridgeHistory( bridge, timestamp) logging.debug("Closing descriptor document") f.close() # read pluggable transports from extra-info document # XXX: should read from networkstatus after bridge-authority # does a reachability test if hasattr(cfg, "EXTRA_INFO_FILE"): logging.info("Opening Extra Info document %s", os.path.abspath(cfg.EXTRA_INFO_FILE)) f = open(cfg.EXTRA_INFO_FILE, 'r') for transport in Bridges.parseExtraInfoFile(f): ID, method_name, address, port, argdict = transport if bridges[ID].running: logging.debug(" Appending transport to running bridge") bridges[ID].transports.append( Bridges.PluggableTransport(bridges[ID], method_name, address, port, argdict)) assert bridges[ ID].transports, "We added a transport but it disappeared!" logging.debug("Closing extra-info document") f.close() if hasattr(cfg, "COUNTRY_BLOCK_FILE"): logging.info("Opening Blocking Countries file %s", os.path.abspath(cfg.COUNTRY_BLOCK_FILE)) f = open(cfg.COUNTRY_BLOCK_FILE, 'r') for ID, address, portlist, countries in Bridges.parseCountryBlockFile( f): if ID in bridges.keys() and bridges[ID].running: for port in portlist: logging.debug(":.( Tears! %s blocked %s %s:%s", countries, bridges[ID].fingerprint, address, port) try: bridges[ID].blockingCountries["%s:%s" % \ (address, port)].update(countries) except KeyError: bridges[ID].blockingCountries["%s:%s" % \ (address, port)] = set(countries) logging.debug("Closing blocking-countries document") f.close() bridges = None