def testDistWithFilterBlockedCountriesAdvanced(self):
d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")
for _ in xrange(250):
d.insert(fakeBridge6(or_addresses=True, transports=True))
d.insert(fakeBridge(or_addresses=True, transports=True))
for b in d.splitter.bridges:
# china blocks some transports
for pt in b.transports:
if random.choice(xrange(2)) > 0:
key = "%s:%s" % (pt.address, pt.port)
b.blockingCountries[key] = set(['cn'])
for address, portlist in b.or_addresses.items():
# china blocks some transports
for port in portlist:
if random.choice(xrange(2)) > 0:
key = "%s:%s" % (address, port)
b.blockingCountries[key] = set(['cn'])
key = "%s:%s" % (b.ip, b.orport)
b.blockingCountries[key] = set(['cn'])
# we probably will get at least one bridge back!
# it's pretty unlikely to lose a coin flip 250 times in a row
for i in xrange(5):
b = d.getBridgesForIP(randomIPString(), "x", 1,
bridgeFilterRules=[
filterBridgesByNotBlockedIn("cn", methodname='obfs2'),
filterBridgesByTransport('obfs2'),
])
try: assert len(b) > 0
except AssertionError:
print "epic fail"
b = d.getBridgesForIP(randomIPString(), "x", 1, bridgeFilterRules=[
filterBridgesByNotBlockedIn("us")])
assert len(b) > 0
def testDistWithFilterBlockedCountriesAdvanced(self):
d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")
for _ in xrange(250):
d.insert(fakeBridge6(or_addresses=True, transports=True))
d.insert(fakeBridge(or_addresses=True, transports=True))
for b in d.splitter.bridges:
# china blocks some transports
for pt in b.transports:
if random.choice(xrange(2)) > 0:
key = "%s:%s" % (pt.address, pt.port)
b.blockingCountries[key] = set(['cn'])
for address, portlist in b.or_addresses.items():
# china blocks some transports
for port in portlist:
if random.choice(xrange(2)) > 0:
key = "%s:%s" % (address, port)
b.blockingCountries[key] = set(['cn'])
key = "%s:%s" % (b.ip, b.orport)
b.blockingCountries[key] = set(['cn'])
# we probably will get at least one bridge back!
# it's pretty unlikely to lose a coin flip 250 times in a row
for i in xrange(5):
b = d.getBridgesForIP(randomIPString(),
"x",
1,
bridgeFilterRules=[
filterBridgesByNotBlockedIn(
"cn", methodname='obfs2'),
filterBridgesByTransport('obfs2'),
])
try:
assert len(b) > 0
except AssertionError:
print("epic fail")
b = d.getBridgesForIP(
randomIPString(),
"x",
1,
bridgeFilterRules=[filterBridgesByNotBlockedIn("us")])
assert len(b) > 0
def getBridgeRequestAnswer(self, request):
"""Respond to a client HTTP request for bridges.
:type request: :api:`twisted.web.http.Request`
:param request: A ``Request`` object containing the HTTP method, full
URI, and any URL/POST arguments and headers present.
:rtype: str
:returns: A plaintext or HTML response to serve.
"""
# XXX why are we getting the interval if our distributor might be
# using bridgedb.schedule.Unscheduled?
interval = self.schedule.intervalStart(time.time())
bridges = ( )
ip = None
countryCode = None
# XXX this code is duplicated in CaptchaProtectedResource
if self.useForwardedHeader:
h = request.getHeader("X-Forwarded-For")
if h:
ip = h.split(",")[-1].strip()
if not bridgedb.Bridges.is_valid_ip(ip):
logging.warn("Got weird forwarded-for value %r",h)
ip = None
else:
ip = request.getClientIP()
# XXX This can also be a separate function
# XXX if the ip is None, this throws an exception
if geoip:
countryCode = geoip.country_code_by_addr(ip)
if countryCode:
logging.debug("Client request from GeoIP CC: %s" % countryCode)
# XXX separate function again
format = request.args.get("format", None)
if format and len(format): format = format[0] # choose the first arg
# do want any options?
transport = ipv6 = unblocked = False
ipv6 = request.args.get("ipv6", False)
if ipv6: ipv6 = True # if anything after ?ipv6=
# XXX oh dear hell. why not check for the '?transport=' arg before
# regex'ing? And why not compile the regex once, somewhere outside
# this function and class?
try:
# validate method name
transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
request.args.get("transport")[0]).group()
except (TypeError, IndexError, AttributeError):
transport = None
try:
unblocked = re.match('[a-zA-Z]{2,4}',
request.args.get("unblocked")[0]).group()
except (TypeError, IndexError, AttributeError):
unblocked = False
logging.info("Replying to web request from %s. Parameters were %r"
% (ip, request.args))
rules = []
bridgeLines = None
if ip:
if ipv6:
rules.append(filterBridgesByIP6)
addressClass = IPv6Address
else:
rules.append(filterBridgesByIP4)
addressClass = IPv4Address
if transport:
#XXX: A cleaner solution would differentiate between
# addresses by protocol rather than have separate lists
# Tor to be a transport, and selecting between them.
rules = [filterBridgesByTransport(transport, addressClass)]
if unblocked:
rules.append(filterBridgesByNotBlockedIn(unblocked,
addressClass, transport))
bridges = self.distributor.getBridgesForIP(ip, interval,
self.nBridgesToGive,
countryCode,
bridgeFilterRules=rules)
bridgeLines = "".join(" %s\n" % b.getConfigLine(
includeFingerprint=self.includeFingerprints,
addressClass=addressClass,
transport=transport,
request=bridgedb.Dist.uniformMap(ip)
) for b in bridges)
answer = self.renderAnswer(request, bridgeLines, format)
return answer
def getBridgeRequestAnswer(self, request):
"""Respond to a client HTTP request for bridges.
:type request: :api:`twisted.web.http.Request`
:param request: A ``Request`` object containing the HTTP method, full
URI, and any URL/POST arguments and headers present.
:rtype: str
:returns: A plaintext or HTML response to serve.
"""
# XXX why are we getting the interval if our distributor might be
# using bridgedb.schedule.Unscheduled?
interval = self.schedule.intervalStart(time.time())
bridges = ( )
ip = None
countryCode = None
# XXX this code is duplicated in CaptchaProtectedResource
if self.useForwardedHeader:
h = request.getHeader("X-Forwarded-For")
if h:
ip = h.split(",")[-1].strip()
if not isIPAddress(ip):
logging.warn("Got weird forwarded-for value %r",h)
ip = None
else:
ip = request.getClientIP()
# Look up the country code of the input IP
if isIPAddress(ip):
countryCode = bridgedb.geo.getCountryCode(IPAddress(ip))
else:
logging.warn("Invalid IP detected; skipping country lookup.")
countryCode = None
# XXX separate function again
format = request.args.get("format", None)
if format and len(format): format = format[0] # choose the first arg
# do want any options?
transport = ipv6 = unblocked = False
ipv6 = request.args.get("ipv6", False)
if ipv6: ipv6 = True # if anything after ?ipv6=
# XXX oh dear hell. why not check for the '?transport=' arg before
# regex'ing? And why not compile the regex once, somewhere outside
# this function and class?
try:
# validate method name
transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
request.args.get("transport")[0]).group()
except (TypeError, IndexError, AttributeError):
transport = None
try:
unblocked = re.match('[a-zA-Z]{2,4}',
request.args.get("unblocked")[0]).group()
except (TypeError, IndexError, AttributeError):
unblocked = False
logging.info("Replying to web request from %s. Parameters were %r"
% (ip, request.args))
rules = []
bridgeLines = None
if ip:
if ipv6:
rules.append(filterBridgesByIP6)
addressClass = IPv6Address
else:
rules.append(filterBridgesByIP4)
addressClass = IPv4Address
if transport:
#XXX: A cleaner solution would differentiate between
# addresses by protocol rather than have separate lists
# Tor to be a transport, and selecting between them.
rules = [filterBridgesByTransport(transport, addressClass)]
if unblocked:
rules.append(filterBridgesByNotBlockedIn(unblocked,
addressClass, transport))
bridges = self.distributor.getBridgesForIP(ip, interval,
self.nBridgesToGive,
countryCode,
bridgeFilterRules=rules)
bridgeLines = "".join("%s\n" % b.getConfigLine(
includeFingerprint=self.includeFingerprints,
addressClass=addressClass,
transport=transport,
request=bridgedb.Dist.uniformMap(ip)
) for b in bridges)
answer = self.renderAnswer(request, bridgeLines, format)
return answer
def getBridgeRequestAnswer(self, request):
""" returns a response to a bridge request """
interval = self.schedule.getInterval(time.time())
bridges = ( )
ip = None
countryCode = None
if self.useForwardedHeader:
h = request.getHeader("X-Forwarded-For")
if h:
ip = h.split(",")[-1].strip()
if not bridgedb.Bridges.is_valid_ip(ip):
logging.warn("Got weird forwarded-for value %r",h)
ip = None
else:
ip = request.getClientIP()
if geoip:
countryCode = geoip.country_code_by_addr(ip)
# set locale
setLocaleFromRequestHeader(request)
format = request.args.get("format", None)
if format and len(format): format = format[0] # choose the first arg
# do want any options?
transport = ipv6 = unblocked = False
ipv6 = request.args.get("ipv6", False)
if ipv6: ipv6 = True # if anything after ?ipv6=
try:
# validate method name
transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
request.args.get("transport")[0]).group()
except (TypeError, IndexError, AttributeError):
transport = None
try:
unblocked = re.match('[a-zA-Z]{2,4}',
request.args.get("unblocked")[0]).group()
except (TypeError, IndexError, AttributeError):
unblocked = False
rules = []
if ip:
if ipv6:
rules.append(filterBridgesByIP6)
addressClass = IPv6Address
else:
rules.append(filterBridgesByIP4)
addressClass = IPv4Address
if transport:
#XXX: A cleaner solution would differentiate between
# addresses by protocol rather than have separate lists
# Tor to be a transport, and selecting between them.
rules = [filterBridgesByTransport(transport, addressClass)]
if unblocked:
rules.append(filterBridgesByNotBlockedIn(unblocked,
addressClass, transport))
bridges = self.distributor.getBridgesForIP(ip, interval,
self.nBridgesToGive,
countryCode,
bridgeFilterRules=rules)
answer = None
if bridges:
answer = "".join(" %s\n" % b.getConfigLine(
includeFingerprint=self.includeFingerprints,
addressClass=addressClass,
transport=transport,
request=bridgedb.Dist.uniformMap(ip)
) for b in bridges)
logging.info("Replying to web request from %s. Parameters were %r",
Util.logSafely(ip), request.args)
if format == 'plain':
request.setHeader("Content-Type", "text/plain")
return answer
else:
request.setHeader("Content-Type", "text/html; charset=utf-8")
return lookup.get_template('bridges.html').render(answer=answer)
def getMailResponse(lines, ctx):
"""Given a list of lines from an incoming email message, and a
MailContext object, parse the email and decide what to do in response.
If we want to answer, return a 2-tuple containing the address that
will receive the response, and a readable filelike object containing
the response. Return None,None if we shouldn't answer.
"""
# Extract data from the headers.
msg = rfc822.Message(MailFile(lines))
subject = msg.getheader("Subject", None)
if not subject: subject = "[no subject]"
clientFromAddr = msg.getaddr("From")
clientSenderAddr = msg.getaddr("Sender")
# RFC822 requires at least one 'To' address
clientToList = msg.getaddrlist("To")
clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
msgID = msg.getheader("Message-ID", None)
if clientSenderAddr and clientSenderAddr[1]:
clientAddr = clientSenderAddr[1]
elif clientFromAddr and clientFromAddr[1]:
clientAddr = clientFromAddr[1]
else:
logging.info("No From or Sender header on incoming mail.")
return None,None
# Look up the locale part in the 'To:' address, if there is one and get
# the appropriate Translation object
lang = getLocaleFromPlusAddr(clientToaddr)
t = I18n.getLang(lang)
try:
_, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
except BadEmail:
logging.info("Ignoring bad address on incoming email.")
return None,None
if not addrdomain:
logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
return None,None
rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
if 'dkim' in rules:
# getheader() returns the last of a given kind of header; we want
# to get the first, so we use getheaders() instead.
dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
dkimHeader = "<no header>"
if dkimHeaders: dkimHeader = dkimHeaders[0]
if not dkimHeader.startswith("pass"):
logging.info("Got a bad dkim header (%r) on an incoming mail; "
"rejecting it.", dkimHeader)
return None, None
# Was the magic string included
#for ln in lines:
# if ln.strip().lower() in ("get bridges", "subject: get bridges"):
# break
#else:
# logging.info("Got a mail from %r with no bridge request; dropping",
# clientAddr)
# return None,None
# Figure out which bridges to send
unblocked = transport = ipv6 = skippedheaders = False
bridgeFilterRules = []
addressClass = None
for ln in lines:
# ignore all lines before the subject header
if "subject" in ln.strip().lower():
skippedheaders = True
if not skippedheaders: continue
if "ipv6" in ln.strip().lower():
ipv6 = True
if "transport" in ln.strip().lower():
try:
transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
ln).group(1).strip()
except (TypeError, AttributeError):
transport = None
logging.debug("Got request for transport: %s" % transport)
if "unblocked" in ln.strip().lower():
try:
unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
ln).group(1).strip()
except (TypeError, AttributeError):
transport = None
if ipv6:
bridgeFilterRules.append(filterBridgesByIP6)
addressClass = IPv6Address
else:
bridgeFilterRules.append(filterBridgesByIP4)
addressClass = IPv4Address
if transport:
bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]
if unblocked:
rules.append(filterBridgesByNotBlockedIn(unblocked,
addressClass, transport))
try:
interval = ctx.schedule.getInterval(time.time())
bridges = ctx.distributor.getBridgesForEmail(clientAddr,
interval, ctx.N,
countryCode=None,
bridgeFilterRules=bridgeFilterRules)
# Handle rate limited email
except TooSoonEmail, e:
logging.info("Got a mail too frequently; warning %r: %s.",
Util.logSafely(clientAddr), e)
# Compose a warning email
# MAX_EMAIL_RATE is in seconds, convert to hours
body = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
gpgContext=ctx.gpgContext)
def getMailResponse(lines, ctx):
"""Given a list of lines from an incoming email message, and a
MailContext object, parse the email and decide what to do in response.
If we want to answer, return a 2-tuple containing the address that
will receive the response, and a readable filelike object containing
the response. Return None,None if we shouldn't answer.
"""
# Extract data from the headers.
msg = rfc822.Message(MailFile(lines))
subject = msg.getheader("Subject", None)
if not subject: subject = "[no subject]"
clientFromAddr = msg.getaddr("From")
clientSenderAddr = msg.getaddr("Sender")
# RFC822 requires at least one 'To' address
clientToList = msg.getaddrlist("To")
clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
msgID = msg.getheader("Message-ID", None)
if clientSenderAddr and clientSenderAddr[1]:
clientAddr = clientSenderAddr[1]
elif clientFromAddr and clientFromAddr[1]:
clientAddr = clientFromAddr[1]
else:
logging.info("No From or Sender header on incoming mail.")
return None,None
# Look up the locale part in the 'To:' address, if there is one and get
# the appropriate Translation object
lang = getLocaleFromPlusAddr(clientToaddr)
t = I18n.getLang(lang)
try:
_, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
except BadEmail:
logging.info("Ignoring bad address on incoming email.")
return None,None
if not addrdomain:
logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
return None,None
rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
if 'dkim' in rules:
# getheader() returns the last of a given kind of header; we want
# to get the first, so we use getheaders() instead.
dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
dkimHeader = "<no header>"
if dkimHeaders: dkimHeader = dkimHeaders[0]
if not dkimHeader.startswith("pass"):
logging.info("Got a bad dkim header (%r) on an incoming mail; "
"rejecting it.", dkimHeader)
return None, None
# Was the magic string included
#for ln in lines:
# if ln.strip().lower() in ("get bridges", "subject: get bridges"):
# break
#else:
# logging.info("Got a mail from %r with no bridge request; dropping",
# clientAddr)
# return None,None
# Figure out which bridges to send
unblocked = transport = ipv6 = skippedheaders = False
bridgeFilterRules = []
addressClass = None
for ln in lines:
# ignore all lines before the subject header
if "subject" in ln.strip().lower():
skippedheaders = True
if not skippedheaders: continue
if "ipv6" in ln.strip().lower():
ipv6 = True
if "transport" in ln.strip().lower():
try:
transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
ln).group(1).strip()
except (TypeError, AttributeError):
transport = None
logging.debug("Got request for transport: %s" % transport)
if "unblocked" in ln.strip().lower():
try:
unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
ln).group(1).strip()
except (TypeError, AttributeError):
transport = None
if ipv6:
bridgeFilterRules.append(filterBridgesByIP6)
addressClass = IPv6Address
else:
bridgeFilterRules.append(filterBridgesByIP4)
addressClass = IPv4Address
if transport:
bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]
if unblocked:
rules.append(filterBridgesByNotBlockedIn(unblocked,
addressClass, transport))
try:
interval = ctx.schedule.getInterval(time.time())
bridges = ctx.distributor.getBridgesForEmail(clientAddr,
interval, ctx.N,
countryCode=None,
bridgeFilterRules=bridgeFilterRules)
# Handle rate limited email
except TooSoonEmail, e:
logging.info("Got a mail too frequently; warning %r: %s.",
Util.logSafely(clientAddr), e)
# Compose a warning email
# MAX_EMAIL_RATE is in seconds, convert to hours
body = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
gpgContext=ctx.gpgContext)