def test_byFilters_bySubring_byTransport_wrong_subhashring_no_transport(self):
"""Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the
Bridge has no transports and is assigned to sub-hashring 1-of-2
should return False.
"""
filtre = filters.byFilters([filters.bySubring(self.hmac, 2, 2),
filters.byTransport('voltron')])
self.assertFalse(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_correct_subhashring_with_transport(self):
"""Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the
Bridge has a voltron transport and is assigned to sub-hashring 1-of-2
should return True.
"""
self.addIPv4VoltronPT()
filtre = filters.byFilters([filters.bySubring(self.hmac, 1, 2),
filters.byTransport('voltron')])
self.assertTrue(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_correct_subhashring_no_transport(
self):
"""Filtering byTransport('voltron') and bySubring(HMAC, 1, 2) when the
Bridge has no transports and is assigned to sub-hashring 1-of-2
should return False.
"""
filtre = filters.byFilters([
filters.bySubring(self.hmac, 1, 2),
filters.byTransport('voltron')
])
self.assertFalse(filtre(self.bridge))
def test_byFilters_bySubring_byTransport_wrong_subhashring_with_transport(
self):
"""Filtering byTransport('voltron') and bySubring(HMAC, 2, 2) when the
Bridge has a voltron transport and is assigned to sub-hashring 1-of-2
should return False.
"""
self.addIPv4VoltronPT()
filtre = filters.byFilters([
filters.bySubring(self.hmac, 2, 2),
filters.byTransport('voltron')
])
self.assertFalse(filtre(self.bridge))
def prepopulateRings(self):
"""Prepopulate this distributor's hashrings and subhashrings with
bridges.
"""
logging.info("Prepopulating %s distributor hashrings..." % self.name)
for filterFn in [byIPv4, byIPv6]:
ruleset = frozenset([filterFn])
key = getHMAC(self.key, "Order-Bridges-In-Ring")
ring = BridgeRing(key, self.answerParameters)
self.hashring.addRing(ring,
ruleset,
byFilters([filterFn]),
populate_from=self.hashring.bridges)
# Since prepopulateRings is called every half hour when the bridge
# descriptors are re-parsed, we should clean the database then.
self.cleanDatabase()
def prepopulateRings(self):
"""Prepopulate this distributor's hashrings and subhashrings with
bridges.
"""
logging.info("Prepopulating %s distributor hashrings..." % self.name)
for filterFn in [byIPv4, byIPv6]:
ruleset = frozenset([filterFn])
key = getHMAC(self.key, "Order-Bridges-In-Ring")
ring = BridgeRing(key, self.answerParameters)
self.hashring.addRing(ring, ruleset, byFilters([filterFn]),
populate_from=self.hashring.bridges)
# Since prepopulateRings is called every half hour when the bridge
# descriptors are re-parsed, we should clean the database then.
self.cleanDatabase()
logging.info("Bridges allotted for %s distribution: %d"
% (self.name, len(self.hashring)))
def test_byFilters_no_filters(self):
self.addIPv4VoltronPT()
filtre = filters.byFilters([])
self.assertTrue(filtre(self.bridge))
def test_byFilters_no_filters(self):
self.addIPv4VoltronPT()
filtre = filters.byFilters([])
self.assertTrue(filtre(self.bridge))
def getBridges(self, bridgeRequest, interval, clock=None):
"""Return a list of bridges to give to a user.
.. hint:: All checks on the email address (which should be stored in
the ``bridgeRequest.client`` attribute), such as checks for
whitelisting and canonicalization of domain name, are done in
:meth:`bridgedb.distributors.email.autoresponder.getMailTo` and
:meth:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`.
:type bridgeRequest:
:class:`~bridgedb.distributors.email.request.EmailBridgeRequest`
:param bridgeRequest: A
:class:`~bridgedb.bridgerequest.BridgeRequestBase` with the
:data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute
set to a string containing the client's full, canonicalized email
address.
:type interval: str
:param interval: The time period when we got this request. This can be
any string, so long as it changes with every period.
:type clock: :api:`twisted.internet.task.Clock`
:param clock: If given, use the clock to ask what time it is, rather
than :api:`time.time`. This should likely only be used for
testing.
:rtype: :any:`list` or ``None``
:returns: A list of :class:`~bridgedb.bridges.Bridges` for the
``bridgeRequest.client``, if allowed. Otherwise, returns ``None``.
"""
if (not bridgeRequest.client) or (bridgeRequest.client == 'default'):
raise addr.BadEmail(
("%s distributor can't get bridges for invalid email address: "
"%s") % (self.name, bridgeRequest.client), bridgeRequest.client)
logging.info("Attempting to get bridges for %s..." % bridgeRequest.client)
now = time.time()
if clock:
now = clock.seconds()
with bridgedb.Storage.getDB() as db:
wasWarned = db.getWarnedEmail(bridgeRequest.client)
lastSaw = db.getEmailTime(bridgeRequest.client)
if lastSaw is not None:
if bridgeRequest.client in self.whitelist:
logging.info(
"Whitelisted address %s was last seen %d seconds ago."
% (bridgeRequest.client, now - lastSaw))
elif (lastSaw + self.emailRateMax) >= now:
wait = (lastSaw + self.emailRateMax) - now
logging.info("Client %s must wait another %d seconds."
% (bridgeRequest.client, wait))
if wasWarned:
raise IgnoreEmail(
"Client %s was warned." % bridgeRequest.client,
bridgeRequest.client)
else:
logging.info("Sending duplicate request warning.")
db.setWarnedEmail(bridgeRequest.client, True, now)
db.commit()
raise TooSoonEmail("Must wait %d seconds" % wait,
bridgeRequest.client)
# warning period is over
elif wasWarned:
db.setWarnedEmail(bridgeRequest.client, False)
pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client))
ring = None
filtres = frozenset(bridgeRequest.filters)
if filtres in self.hashring.filterRings:
logging.debug("Cache hit %s" % filtres)
_, ring = self.hashring.filterRings[filtres]
else:
logging.debug("Cache miss %s" % filtres)
key = getHMAC(self.key, "Order-Bridges-In-Ring")
ring = BridgeRing(key, self.answerParameters)
self.hashring.addRing(ring, filtres, byFilters(filtres),
populate_from=self.hashring.bridges)
returnNum = self.bridgesPerResponse(ring)
result = ring.getBridges(pos, returnNum, filterBySubnet=False)
db.setEmailTime(bridgeRequest.client, now)
db.commit()
return result
def getBridges(self, bridgeRequest, interval, clock=None):
"""Return a list of bridges to give to a user.
.. hint:: All checks on the email address (which should be stored in
the ``bridgeRequest.client`` attribute), such as checks for
whitelisting and canonicalization of domain name, are done in
:meth:`bridgedb.distributors.email.autoresponder.getMailTo` and
:meth:`bridgedb.distributors.email.autoresponder.SMTPAutoresponder.runChecks`.
:type bridgeRequest:
:class:`~bridgedb.distributors.email.request.EmailBridgeRequest`
:param bridgeRequest: A
:class:`~bridgedb.bridgerequest.BridgeRequestBase` with the
:data:`~bridgedb.bridgerequest.BridgeRequestBase.client` attribute
set to a string containing the client's full, canonicalized email
address.
:type interval: str
:param interval: The time period when we got this request. This can be
any string, so long as it changes with every period.
:type clock: :api:`twisted.internet.task.Clock`
:param clock: If given, use the clock to ask what time it is, rather
than :api:`time.time`. This should likely only be used for
testing.
:rtype: :any:`list` or ``None``
:returns: A list of :class:`~bridgedb.bridges.Bridge` for the
``bridgeRequest.client``, if allowed. Otherwise, returns ``None``.
"""
if (not bridgeRequest.client) or (bridgeRequest.client == 'default'):
raise addr.BadEmail(
("%s distributor can't get bridges for invalid email address: "
"%s") % (self.name, bridgeRequest.client),
bridgeRequest.client)
logging.info("Attempting to get bridges for %s..." %
bridgeRequest.client)
now = time.time()
if clock:
now = clock.seconds()
with bridgedb.Storage.getDB() as db:
wasWarned = db.getWarnedEmail(bridgeRequest.client)
lastSaw = db.getEmailTime(bridgeRequest.client)
if lastSaw is not None:
if bridgeRequest.client in self.whitelist:
logging.info(
"Whitelisted address %s was last seen %d seconds ago."
% (bridgeRequest.client, now - lastSaw))
elif (lastSaw + self.emailRateMax) >= now:
wait = (lastSaw + self.emailRateMax) - now
logging.info("Client %s must wait another %d seconds." %
(bridgeRequest.client, wait))
if wasWarned:
raise IgnoreEmail(
"Client %s was warned." % bridgeRequest.client,
bridgeRequest.client)
else:
logging.info("Sending duplicate request warning.")
db.setWarnedEmail(bridgeRequest.client, True, now)
db.commit()
raise TooSoonEmail("Must wait %d seconds" % wait,
bridgeRequest.client)
# warning period is over
elif wasWarned:
db.setWarnedEmail(bridgeRequest.client, False)
pos = self.emailHmac("<%s>%s" % (interval, bridgeRequest.client))
ring = None
filtres = frozenset(bridgeRequest.filters)
if filtres in self.hashring.filterRings:
logging.debug("Cache hit %s" % filtres)
_, ring = self.hashring.filterRings[filtres]
else:
logging.debug("Cache miss %s" % filtres)
key = getHMAC(self.key, "Order-Bridges-In-Ring")
ring = BridgeRing(key, self.answerParameters)
self.hashring.addRing(ring,
filtres,
byFilters(filtres),
populate_from=self.hashring.bridges)
returnNum = self.bridgesPerResponse(ring)
result = ring.getBridges(pos, returnNum, filterBySubnet=False)
db.setEmailTime(bridgeRequest.client, now)
db.commit()
return result
def getBridges(self, bridgeRequest, interval):
"""Return a list of bridges to give to a user.
:type bridgeRequest: :class:`bridgedb.https.request.HTTPSBridgeRequest`
:param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase`
with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client`
attribute set to a string containing the client's IP address.
:param str interval: The time period when we got this request. This
can be any string, so long as it changes with every period.
:rtype: list
:return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in
the response. See
:meth:`bridgedb.https.server.WebResourceBridges.getBridgeRequestAnswer`
for an example of how this is used.
"""
logging.info("Attempting to get bridges for %s..." %
bridgeRequest.client)
if not len(self.hashring):
logging.warn("Bailing! Hashring has zero bridges!")
return []
usingProxy = False
# First, check if the client's IP is one of the known :data:`proxies`:
if bridgeRequest.client in self.proxies:
# The tag is a tag applied to a proxy IP address when it is added
# to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default
# is 'exit_relay'. For other proxies loaded from the
# PROXY_LIST_FILES config option, the default tag is the full
# filename that the IP address originally came from.
usingProxy = True
tag = self.proxies.getTag(bridgeRequest.client)
logging.info("Client was from known proxy (tag: %s): %s" %
(tag, bridgeRequest.client))
subnet = self.getSubnet(bridgeRequest.client, usingProxy)
subring = self.mapSubnetToSubring(subnet, usingProxy)
position = self.mapClientToHashringPosition(interval, subnet)
filters = self._buildHashringFilters(bridgeRequest.filters, subring)
logging.debug("Client request within time interval: %s" % interval)
logging.debug("Assigned client to subhashring %d/%d" %
(subring, self.totalSubrings))
logging.debug("Assigned client to subhashring position: %s" %
position.encode('hex'))
logging.debug("Total bridges: %d" % len(self.hashring))
logging.debug("Bridge filters: %s" %
' '.join([x.func_name for x in filters]))
# Check wheth we have a cached copy of the hashring:
if filters in self.hashring.filterRings.keys():
logging.debug("Cache hit %s" % filters)
_, ring = self.hashring.filterRings[filters]
# Otherwise, construct a new hashring and populate it:
else:
logging.debug("Cache miss %s" % filters)
key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
ring = BridgeRing(key1, self.answerParameters)
self.hashring.addRing(ring,
filters,
byFilters(filters),
populate_from=self.hashring.bridges)
# Determine the appropriate number of bridges to give to the client:
returnNum = self.bridgesPerResponse(ring)
answer = ring.getBridges(position, returnNum)
return answer
def prepopulateRings(self):
"""Prepopulate this distributor's hashrings and subhashrings with
bridges.
The hashring structure for this distributor is influenced by the
``N_IP_CLUSTERS`` configuration option, as well as the number of
``PROXY_LIST_FILES``.
Essentially, :data:`totalSubrings` is set to the specified
``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of
Tor Exit relays (downloaded into memory with :script:`get-tor-exits`),
are stored in :data:`proxies`, and the latter is added as an
additional cluster (such that :data:`totalSubrings` becomes
``N_IP_CLUSTERS + 1``). The number of subhashrings which this
:class:`Distributor` has active in its hashring is then
:data:`totalSubrings`, where the last cluster is reserved for all
:data:`proxies`.
As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and
``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number
of subhashrings is five — four for the "clusters", and one for the
:data:`proxies`. Thus, the resulting hashring-subhashring structure
would look like:
+------------------+---------------------------------------------------+-------------+
| | Directly connecting users | Tor / known |
| | | proxy users |
+------------------+------------+------------+------------+------------+-------------+
| Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 |
+==================+============+============+============+============+=============+
| Subhashrings | | | | | |
| (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) |
+------------------+------------+------------+------------+------------+-------------+
| Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 |
| Subhashrings | | | | | |
| bBy requested +------------+------------+------------+------------+-------------+
| bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 |
| | | | | | |
+------------------+------------+------------+------------+------------+-------------+
The "filtered subhashrings" are essentially filtered copies of their
respective subhashring, such that they only contain bridges which
support IPv4 or IPv6, respectively. Additionally, the contents of
``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint.
Thus, in this example, we end up with **10 total subhashrings**.
"""
logging.info("Prepopulating %s distributor hashrings..." % self.name)
for filterFn in [byIPv4, byIPv6]:
for subring in range(1, self.totalSubrings + 1):
filters = self._buildHashringFilters([
filterFn,
], subring)
key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
ring = BridgeRing(key1, self.answerParameters)
# For consistency with previous implementation of this method,
# only set the "name" for "clusters" which are for this
# distributor's proxies:
if subring == self.proxySubring:
ring.setName('{0} Proxy Ring'.format(self.name))
self.hashring.addRing(ring,
filters,
byFilters(filters),
populate_from=self.hashring.bridges)
def getBridges(self, bridgeRequest, interval):
"""Return a list of bridges to give to a user.
:type bridgeRequest: :class:`bridgedb.distributors.https.request.HTTPSBridgeRequest`
:param bridgeRequest: A :class:`~bridgedb.bridgerequest.BridgeRequestBase`
with the :data:`~bridgedb.bridgerequest.BridgeRequestBase.client`
attribute set to a string containing the client's IP address.
:param str interval: The time period when we got this request. This
can be any string, so long as it changes with every period.
:rtype: list
:return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in
the response. See
:meth:`bridgedb.distributors.https.server.WebResourceBridges.getBridgeRequestAnswer`
for an example of how this is used.
"""
logging.info("Attempting to get bridges for %s..." % bridgeRequest.client)
if not len(self.hashring):
logging.warn("Bailing! Hashring has zero bridges!")
return []
usingProxy = False
# First, check if the client's IP is one of the known :data:`proxies`:
if bridgeRequest.client in self.proxies:
# The tag is a tag applied to a proxy IP address when it is added
# to the bridgedb.proxy.ProxySet. For Tor Exit relays, the default
# is 'exit_relay'. For other proxies loaded from the
# PROXY_LIST_FILES config option, the default tag is the full
# filename that the IP address originally came from.
usingProxy = True
tag = self.proxies.getTag(bridgeRequest.client)
logging.info("Client was from known proxy (tag: %s): %s" %
(tag, bridgeRequest.client))
subnet = self.getSubnet(bridgeRequest.client, usingProxy)
subring = self.mapSubnetToSubring(subnet, usingProxy)
position = self.mapClientToHashringPosition(interval, subnet)
filters = self._buildHashringFilters(bridgeRequest.filters, subring)
logging.debug("Client request within time interval: %s" % interval)
logging.debug("Assigned client to subhashring %d/%d" % (subring, self.totalSubrings))
logging.debug("Assigned client to subhashring position: %s" % position.encode('hex'))
logging.debug("Total bridges: %d" % len(self.hashring))
logging.debug("Bridge filters: %s" % ' '.join([x.func_name for x in filters]))
# Check wheth we have a cached copy of the hashring:
if filters in self.hashring.filterRings.keys():
logging.debug("Cache hit %s" % filters)
_, ring = self.hashring.filterRings[filters]
# Otherwise, construct a new hashring and populate it:
else:
logging.debug("Cache miss %s" % filters)
key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
ring = BridgeRing(key1, self.answerParameters)
self.hashring.addRing(ring, filters, byFilters(filters),
populate_from=self.hashring.bridges)
# Determine the appropriate number of bridges to give to the client:
returnNum = self.bridgesPerResponse(ring)
answer = ring.getBridges(position, returnNum, filterBySubnet=True)
return answer
def prepopulateRings(self):
"""Prepopulate this distributor's hashrings and subhashrings with
bridges.
The hashring structure for this distributor is influenced by the
``N_IP_CLUSTERS`` configuration option, as well as the number of
``PROXY_LIST_FILES``.
Essentially, :data:`totalSubrings` is set to the specified
``N_IP_CLUSTERS``. All of the ``PROXY_LIST_FILES``, plus the list of
Tor Exit relays (downloaded into memory with :script:`get-tor-exits`),
are stored in :data:`proxies`, and the latter is added as an
additional cluster (such that :data:`totalSubrings` becomes
``N_IP_CLUSTERS + 1``). The number of subhashrings which this
:class:`Distributor` has active in its hashring is then
:data:`totalSubrings`, where the last cluster is reserved for all
:data:`proxies`.
As an example, if BridgeDB was configured with ``N_IP_CLUSTERS=4`` and
``PROXY_LIST_FILES=["open-socks-proxies.txt"]``, then the total number
of subhashrings is five — four for the "clusters", and one for the
:data:`proxies`. Thus, the resulting hashring-subhashring structure
would look like:
+------------------+---------------------------------------------------+-------------+
| | Directly connecting users | Tor / known |
| | | proxy users |
+------------------+------------+------------+------------+------------+-------------+
| Clusters | Cluster-1 | Cluster-2 | Cluster-3 | Cluster-4 | Cluster-5 |
+==================+============+============+============+============+=============+
| Subhashrings | | | | | |
| (total, assigned)| (5,1) | (5,2) | (5,3) | (5,4) | (5,5) |
+------------------+------------+------------+------------+------------+-------------+
| Filtered | (5,1)-IPv4 | (5,2)-IPv4 | (5,3)-IPv4 | (5,4)-IPv4 | (5,5)-IPv4 |
| Subhashrings | | | | | |
| bBy requested +------------+------------+------------+------------+-------------+
| bridge type) | (5,1)-IPv6 | (5,2)-IPv6 | (5,3)-IPv6 | (5,4)-IPv6 | (5,5)-IPv6 |
| | | | | | |
+------------------+------------+------------+------------+------------+-------------+
The "filtered subhashrings" are essentially filtered copies of their
respective subhashring, such that they only contain bridges which
support IPv4 or IPv6, respectively. Additionally, the contents of
``(5,1)-IPv4`` and ``(5,1)-IPv6`` sets are *not* disjoint.
Thus, in this example, we end up with **10 total subhashrings**.
"""
logging.info("Prepopulating %s distributor hashrings..." % self.name)
for filterFn in [byIPv4, byIPv6]:
for subring in range(1, self.totalSubrings + 1):
filters = self._buildHashringFilters([filterFn,], subring)
key1 = getHMAC(self.key, "Order-Bridges-In-Ring-%d" % subring)
ring = BridgeRing(key1, self.answerParameters)
# For consistency with previous implementation of this method,
# only set the "name" for "clusters" which are for this
# distributor's proxies:
if subring == self.proxySubring:
ring.setName('{0} Proxy Ring'.format(self.name))
self.hashring.addRing(ring, filters, byFilters(filters),
populate_from=self.hashring.bridges)
logging.info("Bridges allotted for %s distribution: %d"
% (self.name, len(self.hashring)))
logging.info("\tNum bridges:\tFilter set:")
for (ringname, (filterFn, subring)) in self.hashring.filterRings.items():
filterSet = ' '.join(self.hashring.extractFilterNames(ringname))
logging.info("\t%2d bridges\t%s" % (len(subring), filterSet))
logging.info("Total subrings for %s: %d"
% (self.name, len(self.hashring.filterRings)))
