Ejemplo n.º 1
0
def load(cfg, splitter, clear=False):
    """Read all the bridge files from cfg, and pass them into a splitter
       object.
    """
    countryblock = Bridges.CountryBlock()
    if clear:
        logging.info("Clearing old bridges")
        splitter.clear()
        logging.info("Clearing old blocked bridges")
        countryblock.clear() 
    logging.info("Loading bridges")
    status = {}
    if hasattr(cfg, "STATUS_FILE"):
        f = open(cfg.STATUS_FILE, 'r')
        for ID, running, stable in Bridges.parseStatusFile(f):
            status[ID] = running, stable
        f.close()
    if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
        f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
        for fingerprint, countryCode in Bridges.parseCountryBlockFile(f):
            countryblock.insert(fingerprint, countryCode)
        f.close() 
    for fname in cfg.BRIDGE_FILES:
        f = open(fname, 'r')
        for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
            s = status.get(bridge.getID())
            if s is not None:
                running, stable = s
                bridge.setStatus(running=running, stable=stable)
            bridge.setBlockingCountries(
                    countryblock.getBlockingCountries(bridge.fingerprint)) 
            splitter.insert(bridge)
        f.close()
Ejemplo n.º 2
0
def load(cfg, splitter, clear=False):
    """Read all the bridge files from cfg, and pass them into a splitter
       object.
    """
    countryblock = Bridges.CountryBlock()
    if clear:
        logging.info("Clearing old bridges")
        splitter.clear()
        logging.info("Clearing old blocked bridges")
        countryblock.clear()
    logging.info("Loading bridges")
    status = {}
    if hasattr(cfg, "STATUS_FILE"):
        f = open(cfg.STATUS_FILE, 'r')
        for ID, running, stable in Bridges.parseStatusFile(f):
            status[ID] = running, stable
        f.close()
    if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
        f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
        for fingerprint, countryCode in Bridges.parseCountryBlockFile(f):
            countryblock.insert(fingerprint, countryCode)
        f.close()
    for fname in cfg.BRIDGE_FILES:
        f = open(fname, 'r')
        for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
            s = status.get(bridge.getID())
            if s is not None:
                running, stable = s
                bridge.setStatus(running=running, stable=stable)
            bridge.setBlockingCountries(
                countryblock.getBlockingCountries(bridge.fingerprint))
            splitter.insert(bridge)
        f.close()
Ejemplo n.º 3
0
def load(cfg, splitter, clear=False):
    """Read all the bridge files from cfg, and pass them into a splitter
       object.
    """
    if clear:
        logging.info("Clearing old bridges")
        splitter.clear()
    logging.info("Loading bridges")
    status = {}
    addresses = {}
    timestamps = {}
    if hasattr(cfg, "STATUS_FILE"):
        logging.info("Opening Network Status document %s", os.path.abspath(cfg.STATUS_FILE))
        f = open(cfg.STATUS_FILE, "r")
        for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(f):
            status[ID] = running, stable
            addresses[ID] = or_addresses
            if ID in timestamps.keys():
                timestamps[ID].append(timestamp)
            else:
                timestamps[ID] = [timestamp]
            # transports[ID] = transports
        logging.debug("Closing status document")
        f.close()
    bridges = {}
    db = bridgedb.Storage.getDB()
    for fname in cfg.BRIDGE_FILES:
        logging.info("Opening cached Descriptor document %s", fname)
        logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE)
        f = open(fname, "r")
        for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
            if bridge.getID() in bridges:
                logging.warn("Parsed bridge that we've already added. Skipping.")
                logging.debug("  Bridge: %s" % bridge.getID())
                continue
            else:
                bridges[bridge.getID()] = bridge
                s = status.get(bridge.getID())
                if s is not None:
                    running, stable = s
                    bridge.setStatus(running=running, stable=stable)
                bridge.or_addresses = addresses.get(bridge.getID())
                splitter.insert(bridge)
                # add or update BridgeHistory entries into the database
                # XXX: what do we do with all these or_addresses?
                # The bridge stability metrics are only concerned with a single ip:port
                # So for now, we will only consider the bridges primary IP:port
                if bridge.getID() in timestamps.keys():
                    ts = timestamps[bridge.getID()][:]
                    ts.sort()
                    for timestamp in ts:
                        bridgedb.Stability.addOrUpdateBridgeHistory(bridge, timestamp)
        logging.debug("Closing descriptor document")
        f.close()
    # read pluggable transports from extra-info document
    # XXX: should read from networkstatus after bridge-authority
    # does a reachability test
    if hasattr(cfg, "EXTRA_INFO_FILE"):
        logging.info("Opening Extra Info document %s", os.path.abspath(cfg.EXTRA_INFO_FILE))
        f = open(cfg.EXTRA_INFO_FILE, "r")
        for transport in Bridges.parseExtraInfoFile(f):
            ID, method_name, address, port, argdict = transport
            if bridges[ID].running:
                logging.debug("  Appending transport to running bridge")
                bridges[ID].transports.append(
                    Bridges.PluggableTransport(bridges[ID], method_name, address, port, argdict)
                )
                assert bridges[ID].transports, "We added a transport but it disappeared!"
        logging.debug("Closing extra-info document")
        f.close()
    if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
        logging.info("Opening Blocking Countries file %s", os.path.abspath(cfg.COUNTRY_BLOCK_FILE))
        f = open(cfg.COUNTRY_BLOCK_FILE, "r")
        for ID, address, portlist, countries in Bridges.parseCountryBlockFile(f):
            if ID in bridges.keys() and bridges[ID].running:
                for port in portlist:
                    logging.debug(":.( Tears! %s blocked %s %s:%s", countries, bridges[ID].fingerprint, address, port)
                    try:
                        bridges[ID].blockingCountries["%s:%s" % (address, port)].update(countries)
                    except KeyError:
                        bridges[ID].blockingCountries["%s:%s" % (address, port)] = set(countries)
        logging.debug("Closing blocking-countries document")
        f.close()

    bridges = None
Ejemplo n.º 4
0
def load(state, splitter, clear=False):
    """Read and parse all descriptors, and load into a bridge splitter.

    Read all the appropriate bridge files from the saved
    :class:`~bridgedb.persistent.State`, parse and validate them, and then
    store them into our ``state.splitter`` instance. The ``state`` will be
    saved again at the end of this function.

    :type splitter: :class:`BridgeSplitter <bridgedb.Bridges.BridgeHolder>`
    :param splitter: A class which provides a mechanism for HMACing
        Bridges in order to assign them to hashrings.
    :param boolean clear: If True, clear all previous bridges from the
        splitter before parsing for new ones.
    """
    if not state:
        logging.fatal("bridgedb.Main.load() could not retrieve state!")
        sys.exit(2)

    if clear:
        logging.info("Clearing old bridges...")
        splitter.clear()

    logging.info("Loading bridges...")

    bridges = {}
    status = {}
    addresses = {}
    timestamps = {}
    bridges = {}
    desc_digests = {}
    ei_digests = {}

    logging.info("Opening network status file: %s" % state.STATUS_FILE)
    f = open(state.STATUS_FILE, 'r')
    for (ID, nickname, desc_digest, running, stable,
         ORaddr, ORport, or_addresses,
         timestamp) in Bridges.parseStatusFile(f):
        bridge = Bridges.Bridge(nickname, ORaddr, ORport, id_digest=ID,
                                or_addresses=or_addresses)
        bridge.assertOK()
        bridge.setStatus(running, stable)
        bridge.setDescriptorDigest(desc_digest)
        bridges[ID] = bridge

        if ID in timestamps.keys():
            timestamps[ID].append(timestamp)
        else:
            timestamps[ID] = [timestamp]
    logging.debug("Closing network status file")
    f.close()

    for fname in state.BRIDGE_FILES:
        logging.info("Opening bridge-server-descriptor file: '%s'" % fname)
        f = open(fname, 'r')
        desc_digests.update(Bridges.getDescriptorDigests(f))
        if state.COLLECT_TIMESTAMPS:
            for bridge in bridges.values():
                if bridge.getID() in timestamps.keys():
                    ts = timestamps[bridge.getID()][:]
                    ts.sort()
                    for timestamp in ts:
                        logging.debug(
                           "Adding/updating timestamps in BridgeHistory for "\
                           "'%s' in database: %s"
                           % (bridge.fingerprint, timestamp))
                        bridgedb.Stability.addOrUpdateBridgeHistory(
                           bridge, timestamp)
        logging.debug("Closing bridge-server-descriptor file: '%s'" % fname)
        f.close()

    for ID in bridges.keys():
        bridge = bridges[ID]
        if bridge.desc_digest in desc_digests:
            bridge.setVerified()
            bridge.setExtraInfoDigest(desc_digests[bridge.desc_digest])
        # We attempt to insert all bridges. If the bridge is not
        # running, then it is skipped during the insertion process.
        splitter.insert(bridge)

    # read pluggable transports from extra-info document
    # XXX: should read from networkstatus after bridge-authority
    # does a reachability test
    for filename in state.EXTRA_INFO_FILES:
        logging.info("Opening extra-info file: '%s'" % filename)
        f = open(filename, 'r')
        for transport in Bridges.parseExtraInfoFile(f):
            ID, method_name, address, port, argdict = transport
            try:
                if bridges[ID].running:
                    logging.info("Adding %s transport to running bridge"
                                 % method_name)
                    bridgePT = Bridges.PluggableTransport(
                        bridges[ID], method_name, address, port, argdict)
                    bridges[ID].transports.append(bridgePT)
                    if not bridgePT in bridges[ID].transports:
                        logging.critical(
                            "Added a transport, but it disappeared!",
                            "\tTransport: %r" % bridgePT)
            except KeyError as error:
                logging.error("Could not find bridge with fingerprint '%s'."
                              % Bridges.toHex(ID))
        logging.debug("Closing extra-info file: '%s'" % filename)
        f.close()

    if state.COUNTRY_BLOCK_FILE:
        logging.info("Opening Blocking Countries file %s"
                     % state.COUNTRY_BLOCK_FILE)
        f = open(state.COUNTRY_BLOCK_FILE)
        # Identity digest, primary OR address, portlist, country codes
        for ID, addr, portlist, cc in Bridges.parseCountryBlockFile(f):
            if ID in bridges.keys() and bridges[ID].running:
                for port in portlist:
                    addrport = "{0}:{1}".format(addr, port)
                    logging.debug(":'( Tears! %s blocked bridge %s at %s"
                                  % (cc, bridges[ID].fingerprint, addrport))
                    try:
                        bridges[ID].blockingCountries[addrport].update(cc)
                    except KeyError:
                        bridges[ID].blockingCountries[addrport] = set(cc)
        logging.debug("Closing blocking-countries document")
        f.close()

    def updateBridgeHistory(bridges, timestamps):
        if not hasattr(state, 'config'):
            logging.info("updateBridgeHistory(): Config file not set "\
                "in State file.")
            return
        if state.COLLECT_TIMESTAMPS:
            logging.debug("Beginning bridge stability calculations")
            for bridge in bridges.values():
                if bridge.getID() in timestamps.keys():
                    ts = timestamps[bridge.getID()][:]
                    ts.sort()
                    for timestamp in ts:
                        logging.debug(
                            "Updating BridgeHistory timestamps for %s: %s"
                            % (bridge.fingerprint, timestamp))
                        bridgedb.Stability.addOrUpdateBridgeHistory(
                            bridge, timestamp)
            logging.debug("Stability calculations complete")

    reactor.callInThread(updateBridgeHistory, bridges, timestamps)

    bridges = None
    state.save()
    return
Ejemplo n.º 5
0
def load(cfg, splitter, clear=False):
    """Read all the bridge files from cfg, and pass them into a splitter
       object.
    """
    if clear:
        logging.info("Clearing old bridges")
        splitter.clear()
    logging.info("Loading bridges")
    status = {}
    addresses = {}
    timestamps = {}
    if hasattr(cfg, "STATUS_FILE"):
        logging.info("Opening Network Status document %s",
                     os.path.abspath(cfg.STATUS_FILE))
        f = open(cfg.STATUS_FILE, 'r')
        for ID, running, stable, or_addresses, timestamp in Bridges.parseStatusFile(
                f):
            status[ID] = running, stable
            addresses[ID] = or_addresses
            if ID in timestamps.keys(): timestamps[ID].append(timestamp)
            else: timestamps[ID] = [timestamp]
            #transports[ID] = transports
        logging.debug("Closing status document")
        f.close()
    bridges = {}
    db = bridgedb.Storage.getDB()
    for fname in cfg.BRIDGE_FILES:
        logging.info("Opening cached Descriptor document %s", fname)
        logging.debug("Parsing document for purpose=%s", cfg.BRIDGE_PURPOSE)
        f = open(fname, 'r')
        for bridge in Bridges.parseDescFile(f, cfg.BRIDGE_PURPOSE):
            if bridge.getID() in bridges:
                logging.warn(
                    "Parsed bridge that we've already added. Skipping.")
                logging.debug("  Bridge: %s" % bridge.getID())
                continue
            else:
                bridges[bridge.getID()] = bridge
                s = status.get(bridge.getID())
                if s is not None:
                    running, stable = s
                    bridge.setStatus(running=running, stable=stable)
                bridge.or_addresses = addresses.get(bridge.getID())
                splitter.insert(bridge)
                # add or update BridgeHistory entries into the database
                # XXX: what do we do with all these or_addresses?
                # The bridge stability metrics are only concerned with a single ip:port
                # So for now, we will only consider the bridges primary IP:port
                if bridge.getID() in timestamps.keys():
                    ts = timestamps[bridge.getID()][:]
                    ts.sort()
                    for timestamp in ts:
                        bridgedb.Stability.addOrUpdateBridgeHistory(
                            bridge, timestamp)
        logging.debug("Closing descriptor document")
        f.close()
    # read pluggable transports from extra-info document
    # XXX: should read from networkstatus after bridge-authority
    # does a reachability test
    if hasattr(cfg, "EXTRA_INFO_FILE"):
        logging.info("Opening Extra Info document %s",
                     os.path.abspath(cfg.EXTRA_INFO_FILE))
        f = open(cfg.EXTRA_INFO_FILE, 'r')
        for transport in Bridges.parseExtraInfoFile(f):
            ID, method_name, address, port, argdict = transport
            if bridges[ID].running:
                logging.debug("  Appending transport to running bridge")
                bridges[ID].transports.append(
                    Bridges.PluggableTransport(bridges[ID], method_name,
                                               address, port, argdict))
                assert bridges[
                    ID].transports, "We added a transport but it disappeared!"
        logging.debug("Closing extra-info document")
        f.close()
    if hasattr(cfg, "COUNTRY_BLOCK_FILE"):
        logging.info("Opening Blocking Countries file %s",
                     os.path.abspath(cfg.COUNTRY_BLOCK_FILE))
        f = open(cfg.COUNTRY_BLOCK_FILE, 'r')
        for ID, address, portlist, countries in Bridges.parseCountryBlockFile(
                f):
            if ID in bridges.keys() and bridges[ID].running:
                for port in portlist:
                    logging.debug(":.( Tears! %s blocked %s %s:%s", countries,
                                  bridges[ID].fingerprint, address, port)
                    try:
                        bridges[ID].blockingCountries["%s:%s" % \
                                (address, port)].update(countries)
                    except KeyError:
                        bridges[ID].blockingCountries["%s:%s" % \
                                (address, port)] = set(countries)
        logging.debug("Closing blocking-countries document")
        f.close()

    bridges = None