Ejemplo n.º 1
0
    def testDistWithFilterBlockedCountriesAdvanced(self):
        d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")
        for _ in xrange(250):
            d.insert(fakeBridge6(or_addresses=True, transports=True))
            d.insert(fakeBridge(or_addresses=True, transports=True))

        for b in d.splitter.bridges:
            # china blocks some transports
            for pt in b.transports:
                if random.choice(xrange(2)) > 0:
                    key = "%s:%s" % (pt.address, pt.port)
                    b.blockingCountries[key] = set(['cn'])
            for address, portlist in b.or_addresses.items():
                # china blocks some transports
                for port in portlist:
                    if random.choice(xrange(2)) > 0:
                        key = "%s:%s" % (address, port)
                        b.blockingCountries[key] = set(['cn'])
            key = "%s:%s" % (b.ip, b.orport)
            b.blockingCountries[key] = set(['cn'])

        # we probably will get at least one bridge back!
        # it's pretty unlikely to lose a coin flip 250 times in a row
        for i in xrange(5):
            b = d.getBridgesForIP(randomIPString(), "x", 1,
                    bridgeFilterRules=[
                        filterBridgesByNotBlockedIn("cn", methodname='obfs2'),
                        filterBridgesByTransport('obfs2'),
                        ])
            try: assert len(b) > 0
            except AssertionError:
                print "epic fail"
            b = d.getBridgesForIP(randomIPString(), "x", 1, bridgeFilterRules=[
                filterBridgesByNotBlockedIn("us")])
            assert len(b) > 0
Ejemplo n.º 2
0
    def testDistWithFilterBlockedCountriesAdvanced(self):
        d = bridgedb.Dist.IPBasedDistributor(self.dumbAreaMapper, 3, "Foo")
        for _ in xrange(250):
            d.insert(fakeBridge6(or_addresses=True, transports=True))
            d.insert(fakeBridge(or_addresses=True, transports=True))

        for b in d.splitter.bridges:
            # china blocks some transports
            for pt in b.transports:
                if random.choice(xrange(2)) > 0:
                    key = "%s:%s" % (pt.address, pt.port)
                    b.blockingCountries[key] = set(['cn'])
            for address, portlist in b.or_addresses.items():
                # china blocks some transports
                for port in portlist:
                    if random.choice(xrange(2)) > 0:
                        key = "%s:%s" % (address, port)
                        b.blockingCountries[key] = set(['cn'])
            key = "%s:%s" % (b.ip, b.orport)
            b.blockingCountries[key] = set(['cn'])

        # we probably will get at least one bridge back!
        # it's pretty unlikely to lose a coin flip 250 times in a row
        for i in xrange(5):
            b = d.getBridgesForIP(randomIPString(),
                                  "x",
                                  1,
                                  bridgeFilterRules=[
                                      filterBridgesByNotBlockedIn(
                                          "cn", methodname='obfs2'),
                                      filterBridgesByTransport('obfs2'),
                                  ])
            try:
                assert len(b) > 0
            except AssertionError:
                print("epic fail")
            b = d.getBridgesForIP(
                randomIPString(),
                "x",
                1,
                bridgeFilterRules=[filterBridgesByNotBlockedIn("us")])
            assert len(b) > 0
Ejemplo n.º 3
0
    def getBridgeRequestAnswer(self, request):
        """Respond to a client HTTP request for bridges.

        :type request: :api:`twisted.web.http.Request`
        :param request: A ``Request`` object containing the HTTP method, full
            URI, and any URL/POST arguments and headers present.
        :rtype: str
        :returns: A plaintext or HTML response to serve.
        """
        # XXX why are we getting the interval if our distributor might be
        # using bridgedb.schedule.Unscheduled?
        interval = self.schedule.intervalStart(time.time())
        bridges = ( )
        ip = None
        countryCode = None

        # XXX this code is duplicated in CaptchaProtectedResource
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        # XXX This can also be a separate function
        # XXX if the ip is None, this throws an exception
        if geoip:
            countryCode = geoip.country_code_by_addr(ip)
            if countryCode:
                logging.debug("Client request from GeoIP CC: %s" % countryCode)

        # XXX separate function again
        format = request.args.get("format", None)
        if format and len(format): format = format[0] # choose the first arg

        # do want any options?
        transport = ipv6 = unblocked = False

        ipv6 = request.args.get("ipv6", False)
        if ipv6: ipv6 = True # if anything after ?ipv6=

        # XXX oh dear hell. why not check for the '?transport=' arg before
        # regex'ing? And why not compile the regex once, somewhere outside
        # this function and class?
        try:
            # validate method name
            transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
                    request.args.get("transport")[0]).group()
        except (TypeError, IndexError, AttributeError):
            transport = None

        try:
            unblocked = re.match('[a-zA-Z]{2,4}',
                    request.args.get("unblocked")[0]).group()
        except (TypeError, IndexError, AttributeError):
            unblocked = False

        logging.info("Replying to web request from %s. Parameters were %r"
                     % (ip, request.args))

        rules = []
        bridgeLines = None

        if ip:
            if ipv6:
                rules.append(filterBridgesByIP6)
                addressClass = IPv6Address
            else:
                rules.append(filterBridgesByIP4)
                addressClass = IPv4Address

            if transport:
                #XXX: A cleaner solution would differentiate between
                # addresses by protocol rather than have separate lists
                # Tor to be a transport, and selecting between them.
                rules = [filterBridgesByTransport(transport, addressClass)]

            if unblocked:
                rules.append(filterBridgesByNotBlockedIn(unblocked,
                    addressClass, transport))

            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive,
                                                       countryCode,
                                                       bridgeFilterRules=rules)
            bridgeLines = "".join("  %s\n" % b.getConfigLine(
                includeFingerprint=self.includeFingerprints,
                addressClass=addressClass,
                transport=transport,
                request=bridgedb.Dist.uniformMap(ip)
                ) for b in bridges)

        answer = self.renderAnswer(request, bridgeLines, format)
        return answer
Ejemplo n.º 4
0
    def getBridgeRequestAnswer(self, request):
        """Respond to a client HTTP request for bridges.

        :type request: :api:`twisted.web.http.Request`
        :param request: A ``Request`` object containing the HTTP method, full
            URI, and any URL/POST arguments and headers present.
        :rtype: str
        :returns: A plaintext or HTML response to serve.
        """
        # XXX why are we getting the interval if our distributor might be
        # using bridgedb.schedule.Unscheduled?
        interval = self.schedule.intervalStart(time.time())
        bridges = ( )
        ip = None
        countryCode = None

        # XXX this code is duplicated in CaptchaProtectedResource
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not isIPAddress(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        # Look up the country code of the input IP
        if isIPAddress(ip):
            countryCode = bridgedb.geo.getCountryCode(IPAddress(ip))
        else:
            logging.warn("Invalid IP detected; skipping country lookup.")
            countryCode = None

        # XXX separate function again
        format = request.args.get("format", None)
        if format and len(format): format = format[0] # choose the first arg

        # do want any options?
        transport = ipv6 = unblocked = False

        ipv6 = request.args.get("ipv6", False)
        if ipv6: ipv6 = True # if anything after ?ipv6=

        # XXX oh dear hell. why not check for the '?transport=' arg before
        # regex'ing? And why not compile the regex once, somewhere outside
        # this function and class?
        try:
            # validate method name
            transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
                    request.args.get("transport")[0]).group()
        except (TypeError, IndexError, AttributeError):
            transport = None

        try:
            unblocked = re.match('[a-zA-Z]{2,4}',
                    request.args.get("unblocked")[0]).group()
        except (TypeError, IndexError, AttributeError):
            unblocked = False

        logging.info("Replying to web request from %s. Parameters were %r"
                     % (ip, request.args))

        rules = []
        bridgeLines = None

        if ip:
            if ipv6:
                rules.append(filterBridgesByIP6)
                addressClass = IPv6Address
            else:
                rules.append(filterBridgesByIP4)
                addressClass = IPv4Address

            if transport:
                #XXX: A cleaner solution would differentiate between
                # addresses by protocol rather than have separate lists
                # Tor to be a transport, and selecting between them.
                rules = [filterBridgesByTransport(transport, addressClass)]

            if unblocked:
                rules.append(filterBridgesByNotBlockedIn(unblocked,
                    addressClass, transport))

            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive,
                                                       countryCode,
                                                       bridgeFilterRules=rules)
            bridgeLines = "".join("%s\n" % b.getConfigLine(
                includeFingerprint=self.includeFingerprints,
                addressClass=addressClass,
                transport=transport,
                request=bridgedb.Dist.uniformMap(ip)
                ) for b in bridges)

        answer = self.renderAnswer(request, bridgeLines, format)
        return answer
Ejemplo n.º 5
0
    def getBridgeRequestAnswer(self, request):
        """ returns a response to a bridge request """
        interval = self.schedule.getInterval(time.time())
        bridges = ( )
        ip = None
        countryCode = None
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        if geoip:
            countryCode = geoip.country_code_by_addr(ip)

        # set locale
        setLocaleFromRequestHeader(request)

        format = request.args.get("format", None)
        if format and len(format): format = format[0] # choose the first arg

        # do want any options?
        transport = ipv6 = unblocked = False

        ipv6 = request.args.get("ipv6", False)
        if ipv6: ipv6 = True # if anything after ?ipv6=

        try:
            # validate method name
            transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
                    request.args.get("transport")[0]).group()
        except (TypeError, IndexError, AttributeError):
            transport = None

        try:
            unblocked = re.match('[a-zA-Z]{2,4}',
                    request.args.get("unblocked")[0]).group()
        except (TypeError, IndexError, AttributeError):
            unblocked = False

        rules = []

        if ip:
            if ipv6:
                rules.append(filterBridgesByIP6)
                addressClass = IPv6Address
            else:
                rules.append(filterBridgesByIP4)
                addressClass = IPv4Address

            if transport:
                #XXX: A cleaner solution would differentiate between
                # addresses by protocol rather than have separate lists
                # Tor to be a transport, and selecting between them.
                rules = [filterBridgesByTransport(transport, addressClass)]

            if unblocked:
                rules.append(filterBridgesByNotBlockedIn(unblocked,
                    addressClass, transport))

            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive,
                                                       countryCode,
                                                       bridgeFilterRules=rules)

        answer = None
        if bridges:
            answer = "".join("  %s\n" % b.getConfigLine(
                includeFingerprint=self.includeFingerprints,
                addressClass=addressClass,
                transport=transport,
                request=bridgedb.Dist.uniformMap(ip)
                ) for b in bridges) 

        logging.info("Replying to web request from %s.  Parameters were %r",
                     Util.logSafely(ip), request.args)
        if format == 'plain':
            request.setHeader("Content-Type", "text/plain")
            return answer
        else:
            request.setHeader("Content-Type", "text/html; charset=utf-8")
            return lookup.get_template('bridges.html').render(answer=answer)
Ejemplo n.º 6
0
def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    # RFC822 requires at least one 'To' address
    clientToList = msg.getaddrlist("To")
    clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
    msgID = msg.getheader("Message-ID", None)
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    # Look up the locale part in the 'To:' address, if there is one and get
    # the appropriate Translation object
    lang = getLocaleFromPlusAddr(clientToaddr)
    t = I18n.getLang(lang)

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except BadEmail:
        logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
        logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
        return None,None
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    #for ln in lines:
    #    if ln.strip().lower() in ("get bridges", "subject: get bridges"):
    #        break
    #else:
    #    logging.info("Got a mail from %r with no bridge request; dropping",
    #                 clientAddr)
    #    return None,None

    # Figure out which bridges to send
    unblocked = transport = ipv6 = skippedheaders = False
    bridgeFilterRules = []
    addressClass = None
    for ln in lines:
        # ignore all lines before the subject header
        if "subject" in ln.strip().lower():
            skippedheaders = True
        if not skippedheaders: continue

        if "ipv6" in ln.strip().lower():
            ipv6 = True
        if "transport" in ln.strip().lower():
            try:
                transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None
            logging.debug("Got request for transport: %s" % transport)
        if "unblocked" in ln.strip().lower():
            try:
                unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None

    if ipv6:
        bridgeFilterRules.append(filterBridgesByIP6)
        addressClass = IPv6Address
    else:
        bridgeFilterRules.append(filterBridgesByIP4)
        addressClass = IPv4Address

    if transport:
        bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]

    if unblocked:
        rules.append(filterBridgesByNotBlockedIn(unblocked,
            addressClass, transport))

    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
            interval, ctx.N,
            countryCode=None,
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)
Ejemplo n.º 7
0
def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    # RFC822 requires at least one 'To' address
    clientToList = msg.getaddrlist("To")
    clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
    msgID = msg.getheader("Message-ID", None)
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    # Look up the locale part in the 'To:' address, if there is one and get
    # the appropriate Translation object
    lang = getLocaleFromPlusAddr(clientToaddr)
    t = I18n.getLang(lang)

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except BadEmail:
        logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
        logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
        return None,None
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    #for ln in lines:
    #    if ln.strip().lower() in ("get bridges", "subject: get bridges"):
    #        break
    #else:
    #    logging.info("Got a mail from %r with no bridge request; dropping",
    #                 clientAddr)
    #    return None,None

    # Figure out which bridges to send
    unblocked = transport = ipv6 = skippedheaders = False
    bridgeFilterRules = []
    addressClass = None
    for ln in lines:
        # ignore all lines before the subject header
        if "subject" in ln.strip().lower():
            skippedheaders = True
        if not skippedheaders: continue

        if "ipv6" in ln.strip().lower():
            ipv6 = True
        if "transport" in ln.strip().lower():
            try:
                transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None
            logging.debug("Got request for transport: %s" % transport)
        if "unblocked" in ln.strip().lower():
            try:
                unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None

    if ipv6:
        bridgeFilterRules.append(filterBridgesByIP6)
        addressClass = IPv6Address
    else:
        bridgeFilterRules.append(filterBridgesByIP4)
        addressClass = IPv4Address

    if transport:
        bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]

    if unblocked:
        rules.append(filterBridgesByNotBlockedIn(unblocked,
            addressClass, transport))

    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
            interval, ctx.N,
            countryCode=None,
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)