def test_byFilters_bySubring_byTransport_wrong_subhashring_no_transport(self): """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the Bridge has no transports and is assigned to sub-hashring 1-of-2 should return False. """ filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2), filters.byTransport('voltron')]) self.assertFalse(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_correct_subhashring_with_transport(self): """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 should return True. """ self.addIPv4VoltronPT() filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2), filters.byTransport('voltron')]) self.assertTrue(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_correct_subhashring_no_transport( self): """Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the Bridge has no transports and is assigned to sub-hashring 1-of-2 should return False. """ filtre = filters.byFilters([ filters.bySubring(self.hmac, 1, 2), filters.byTransport('voltron') ]) self.assertFalse(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_wrong_subhashring_with_transport( self): """Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the Bridge has a voltron transport and is assigned to sub-hashring 1-of-2 should return False. """ self.addIPv4VoltronPT() filtre = filters.byFilters([ filters.bySubring(self.hmac, 2, 2), filters.byTransport('voltron') ]) self.assertFalse(filtre(self.bridge))
def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [byIPv4, byIPv6]: ruleset = frozenset([filterFn]) key = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key, self.answerParameters) self.hashring.addRing(ring, ruleset, byFilters([filterFn]), populate_from=self.hashring.bridges) # Since prepopulateRings is called every half hour when the bridge # descriptors are re-parsed, we should clean the database then. self.cleanDatabase()
def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [byIPv4, byIPv6]: ruleset = frozenset([filterFn]) key = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key, self.answerParameters) self.hashring.addRing(ring, ruleset, byFilters([filterFn]), populate_from=self.hashring.bridges) # Since prepopulateRings is called every half hour when the bridge # descriptors are re-parsed, we should clean the database then. self.cleanDatabase() logging.info("Bridges allotted for %s distribution: %d" % (self.name, len(self.hashring)))
def test_byFilters_no_filters(self): self.addIPv4VoltronPT() filtre = filters.byFilters([]) self.assertTrue(filtre(self.bridge))
def getBridges(self, bridgeRequest, interval, clock=None): """Return a list of bridges to give to a user. .. hint:: All checks on the email address (which should be stored in the ``bridgeRequest.client`` attribute), such as checks for whitelisting and canonicalization of domain name, are done in :meth:`bridgedb.distributors.email.autoresponder.getMailTo` and :meth:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :type bridgeRequest: :class:`~bridgedb.distributors.email.request.EmailBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's full, canonicalized email address. :type interval: str :param interval: The time period when we got this request. This can be any string, so long as it changes with every period. :type clock: :api:`twisted.internet.task.Clock` :param clock: If given, use the clock to ask what time it is, rather than :api:`time.time`. This should likely only be used for testing. :rtype: :any:`list` or ``None`` :returns: A list of :class:`~bridgedb.bridges.Bridges` for the ``bridgeRequest.client``, if allowed. Otherwise, returns ``None``. """ if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): raise addr.BadEmail( ("%s distributor can't get bridges for invalid email address: " "%s") % (self.name, bridgeRequest.client), bridgeRequest.client) logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) now = time.time() if clock: now = clock.seconds() with bridgedb.Storage.getDB() as db: wasWarned = db.getWarnedEmail(bridgeRequest.client) lastSaw = db.getEmailTime(bridgeRequest.client) if lastSaw is not None: if bridgeRequest.client in self.whitelist: logging.info( "Whitelisted address %s was last seen %d seconds ago." % (bridgeRequest.client, now - lastSaw)) elif (lastSaw + self.emailRateMax) >= now: wait = (lastSaw + self.emailRateMax) - now logging.info("Client %s must wait another %d seconds." % (bridgeRequest.client, wait)) if wasWarned: raise IgnoreEmail( "Client %s was warned." % bridgeRequest.client, bridgeRequest.client) else: logging.info("Sending duplicate request warning.") db.setWarnedEmail(bridgeRequest.client, True, now) db.commit() raise TooSoonEmail("Must wait %d seconds" % wait, bridgeRequest.client) # warning period is over elif wasWarned: db.setWarnedEmail(bridgeRequest.client, False) pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) ring = None filtres = frozenset(bridgeRequest.filters) if filtres in self.hashring.filterRings: logging.debug("Cache hit %s" % filtres) _, ring = self.hashring.filterRings[filtres] else: logging.debug("Cache miss %s" % filtres) key = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key, self.answerParameters) self.hashring.addRing(ring, filtres, byFilters(filtres), populate_from=self.hashring.bridges) returnNum = self.bridgesPerResponse(ring) result = ring.getBridges(pos, returnNum, filterBySubnet=False) db.setEmailTime(bridgeRequest.client, now) db.commit() return result
def getBridges(self, bridgeRequest, interval, clock=None): """Return a list of bridges to give to a user. .. hint:: All checks on the email address (which should be stored in the ``bridgeRequest.client`` attribute), such as checks for whitelisting and canonicalization of domain name, are done in :meth:`bridgedb.distributors.email.autoresponder.getMailTo` and :meth:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`. :type bridgeRequest: :class:`~bridgedb.distributors.email.request.EmailBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's full, canonicalized email address. :type interval: str :param interval: The time period when we got this request. This can be any string, so long as it changes with every period. :type clock: :api:`twisted.internet.task.Clock` :param clock: If given, use the clock to ask what time it is, rather than :api:`time.time`. This should likely only be used for testing. :rtype: :any:`list` or ``None`` :returns: A list of :class:`~bridgedb.bridges.Bridge` for the ``bridgeRequest.client``, if allowed. Otherwise, returns ``None``. """ if (not bridgeRequest.client) or (bridgeRequest.client == 'default'): raise addr.BadEmail( ("%s distributor can't get bridges for invalid email address: " "%s") % (self.name, bridgeRequest.client), bridgeRequest.client) logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) now = time.time() if clock: now = clock.seconds() with bridgedb.Storage.getDB() as db: wasWarned = db.getWarnedEmail(bridgeRequest.client) lastSaw = db.getEmailTime(bridgeRequest.client) if lastSaw is not None: if bridgeRequest.client in self.whitelist: logging.info( "Whitelisted address %s was last seen %d seconds ago." % (bridgeRequest.client, now - lastSaw)) elif (lastSaw + self.emailRateMax) >= now: wait = (lastSaw + self.emailRateMax) - now logging.info("Client %s must wait another %d seconds." % (bridgeRequest.client, wait)) if wasWarned: raise IgnoreEmail( "Client %s was warned." % bridgeRequest.client, bridgeRequest.client) else: logging.info("Sending duplicate request warning.") db.setWarnedEmail(bridgeRequest.client, True, now) db.commit() raise TooSoonEmail("Must wait %d seconds" % wait, bridgeRequest.client) # warning period is over elif wasWarned: db.setWarnedEmail(bridgeRequest.client, False) pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client)) ring = None filtres = frozenset(bridgeRequest.filters) if filtres in self.hashring.filterRings: logging.debug("Cache hit %s" % filtres) _, ring = self.hashring.filterRings[filtres] else: logging.debug("Cache miss %s" % filtres) key = getHMAC(self.key, "Order-Bridges-In-Ring") ring = BridgeRing(key, self.answerParameters) self.hashring.addRing(ring, filtres, byFilters(filtres), populate_from=self.hashring.bridges) returnNum = self.bridgesPerResponse(ring) result = ring.getBridges(pos, returnNum, filterBySubnet=False) db.setEmailTime(bridgeRequest.client, now) db.commit() return result
def getBridges(self, bridgeRequest, interval): """Return a list of bridges to give to a user. :type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's IP address. :param str interval: The time period when we got this request. This can be any string, so long as it changes with every period. :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in the response. See :meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer` for an example of how this is used. """ logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) if not len(self.hashring): logging.warn("Bailing! Hashring has zero bridges!") return [] usingProxy = False # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: # The tag is a tag applied to a proxy IP address when it is added # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default # is 'exit_relay'. For other proxies loaded from the # PROXY_LIST_FILES config option, the default tag is the full # filename that the IP address originally came from. usingProxy = True tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) subnet = self.getSubnet(bridgeRequest.client, usingProxy) subring = self.mapSubnetToSubring(subnet, usingProxy) position = self.mapClientToHashringPosition(interval, subnet) filters = self._buildHashringFilters(bridgeRequest.filters, subring) logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) logging.debug("Total bridges: %d" % len(self.hashring)) logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % filters) _, ring = self.hashring.filterRings[filters] # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: returnNum = self.bridgesPerResponse(ring) answer = ring.getBridges(position, returnNum) return answer
def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. The hashring structure for this distributor is influenced by the ``N_IP_CLUSTERS`` configuration option, as well as the number of ``PROXY_LIST_FILES``. Essentially, :data:`totalSubrings` is set to the specified ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), are stored in :data:`proxies`, and the latter is added as an additional cluster (such that :data:`totalSubrings` becomes ``N_IP_CLUSTERS + 1``). The number of subhashrings which this :class:`Distributor` has active in its hashring is then :data:`totalSubrings`, where the last cluster is reserved for all :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number of subhashrings is five — four for the "clusters", and one for the :data:`proxies`. Thus, the resulting hashring-subhashring structure would look like: +------------------+---------------------------------------------------+-------------+ | | Directly connecting users | Tor / known | | | | proxy users | +------------------+------------+------------+------------+------------+-------------+ | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | +==================+============+============+============+============+=============+ | Subhashrings | | | | | | | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | +------------------+------------+------------+------------+------------+-------------+ | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | | Subhashrings | | | | | | | bBy requested +------------+------------+------------+------------+-------------+ | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | | | | | | | | +------------------+------------+------------+------------+------------+-------------+ The "filtered subhashrings" are essentially filtered copies of their respective subhashring, such that they only contain bridges which support IPv4 or IPv6, respectively. Additionally, the contents of ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. Thus, in this example, we end up with **10 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([ filterFn, ], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges)
def getBridges(self, bridgeRequest, interval): """Return a list of bridges to give to a user. :type bridgeRequest: :class:`bridgedb.distributors.https.request.HTTPSBridgeRequest` :param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase` with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute set to a string containing the client's IP address. :param str interval: The time period when we got this request. This can be any string, so long as it changes with every period. :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in the response. See :meth:`bridgedb.distributors.https.server.WebResourceBridges.getBridgeRequestAnswer` for an example of how this is used. """ logging.info("Attempting to get bridges for %s..." % bridgeRequest.client) if not len(self.hashring): logging.warn("Bailing! Hashring has zero bridges!") return [] usingProxy = False # First, check if the client's IP is one of the known :data:`proxies`: if bridgeRequest.client in self.proxies: # The tag is a tag applied to a proxy IP address when it is added # to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default # is 'exit_relay'. For other proxies loaded from the # PROXY_LIST_FILES config option, the default tag is the full # filename that the IP address originally came from. usingProxy = True tag = self.proxies.getTag(bridgeRequest.client) logging.info("Client was from known proxy (tag: %s): %s" % (tag, bridgeRequest.client)) subnet = self.getSubnet(bridgeRequest.client, usingProxy) subring = self.mapSubnetToSubring(subnet, usingProxy) position = self.mapClientToHashringPosition(interval, subnet) filters = self._buildHashringFilters(bridgeRequest.filters, subring) logging.debug("Client request within time interval: %s" % interval) logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings)) logging.debug("Assigned client to subhashring position: %s" % position.encode('hex')) logging.debug("Total bridges: %d" % len(self.hashring)) logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters])) # Check wheth we have a cached copy of the hashring: if filters in self.hashring.filterRings.keys(): logging.debug("Cache hit %s" % filters) _, ring = self.hashring.filterRings[filters] # Otherwise, construct a new hashring and populate it: else: logging.debug("Cache miss %s" % filters) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) # Determine the appropriate number of bridges to give to the client: returnNum = self.bridgesPerResponse(ring) answer = ring.getBridges(position, returnNum, filterBySubnet=True) return answer
def prepopulateRings(self): """Prepopulate this distributor's hashrings and subhashrings with bridges. The hashring structure for this distributor is influenced by the ``N_IP_CLUSTERS`` configuration option, as well as the number of ``PROXY_LIST_FILES``. Essentially, :data:`totalSubrings` is set to the specified ``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of Tor Exit relays (downloaded into memory with :script:`get-tor-exits`), are stored in :data:`proxies`, and the latter is added as an additional cluster (such that :data:`totalSubrings` becomes ``N_IP_CLUSTERS + 1``). The number of subhashrings which this :class:`Distributor` has active in its hashring is then :data:`totalSubrings`, where the last cluster is reserved for all :data:`proxies`. As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and ``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number of subhashrings is five — four for the "clusters", and one for the :data:`proxies`. Thus, the resulting hashring-subhashring structure would look like: +------------------+---------------------------------------------------+-------------+ | | Directly connecting users | Tor / known | | | | proxy users | +------------------+------------+------------+------------+------------+-------------+ | Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 | +==================+============+============+============+============+=============+ | Subhashrings | | | | | | | (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) | +------------------+------------+------------+------------+------------+-------------+ | Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 | | Subhashrings | | | | | | | bBy requested +------------+------------+------------+------------+-------------+ | bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 | | | | | | | | +------------------+------------+------------+------------+------------+-------------+ The "filtered subhashrings" are essentially filtered copies of their respective subhashring, such that they only contain bridges which support IPv4 or IPv6, respectively. Additionally, the contents of ``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint. Thus, in this example, we end up with **10 total subhashrings**. """ logging.info("Prepopulating %s distributor hashrings..." % self.name) for filterFn in [byIPv4, byIPv6]: for subring in range(1, self.totalSubrings + 1): filters = self._buildHashringFilters([filterFn,], subring) key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring) ring = BridgeRing(key1, self.answerParameters) # For consistency with previous implementation of this method, # only set the "name" for "clusters" which are for this # distributor's proxies: if subring == self.proxySubring: ring.setName('{0} Proxy Ring'.format(self.name)) self.hashring.addRing(ring, filters, byFilters(filters), populate_from=self.hashring.bridges) logging.info("Bridges allotted for %s distribution: %d" % (self.name, len(self.hashring))) logging.info("\tNum bridges:\tFilter set:") for (ringname, (filterFn, subring)) in self.hashring.filterRings.items(): filterSet = ' '.join(self.hashring.extractFilterNames(ringname)) logging.info("\t%2d bridges\t%s" % (len(subring), filterSet)) logging.info("Total subrings for %s: %d" % (self.name, len(self.hashring.filterRings)))