Exemple #1
0
def configureLogging(cfg):
    """Set up Python's logging subsystem based on the configuratino.
    """

    # Turn on safe logging by default
    safelogging = getattr(cfg, "SAFELOGGING", True)

    level = getattr(cfg, "LOGLEVEL", "WARNING")
    level = getattr(logging, level)
    logfile = getattr(cfg, "LOGFILE", "")
    logfile_count = getattr(cfg, "LOGFILE_COUNT", 5)
    logfile_rotate_size = getattr(cfg, "LOGFILE_ROTATE_SIZE", 10000000)
    Util.set_safe_logging(safelogging)

    logging.getLogger().setLevel(level)
    if logfile:
        handler = logging.handlers.RotatingFileHandler(logfile, "a", logfile_rotate_size, logfile_count)
        formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", "%b %d %H:%M:%S")
        handler.setFormatter(formatter)
        logging.getLogger().addHandler(handler)

    logging.info("Logger Started.")
    logging.info("Level: %s", level)
    if logfile:
        logging.info("Log File: %s", os.path.abspath(logfile))
        logging.info("Log File Count: %d", logfile_count)
        logging.info("Rotate Logs After Size: %d", logfile_rotate_size)
    else:
        logging.info("Logging to stderr")
    if safelogging:
        logging.info("Safe Logging: Enabled")
    else:
        logging.warn("Safe Logging: Disabled")
Exemple #2
0
def replyToMail(lines, ctx):
    """Reply to an incoming email. Maybe.

    If no `response` is returned from :func:`getMailResponse`, then the
    incoming email will not be responded to at all. This can happen for
    several reasons, for example: if the DKIM signature was invalid or
    missing, or if the incoming email came from an unacceptable domain, or if
    there have been too many emails from this client in the allotted time
    period.

    :param list lines: A list of lines from an incoming email message.
    :type ctx: :class:`MailContext`
    :param ctx: The configured context for the email server.
    """
    logging.info("Got an email; deciding whether to reply.")
    sendToUser, response = getMailResponse(lines, ctx)

    if response is None:
        logging.debug("getMailResponse() said not to reply to %s, so I won't."
                      % Util.logSafely(sendToUser))
        return

    response.seek(0)
    logging.info("Sending reply to %r", Util.logSafely(sendToUser))

    d = Deferred()
    factory = twisted.mail.smtp.SMTPSenderFactory(ctx.smtpFromAddr, sendToUser,
                                                  response, d)
    reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)

    return d
Exemple #3
0
    def checkSolution(self, request):
        """Process a solved CAPTCHA by sending rehashing the solution together with
        the client's IP address, and checking that the result matches the challenge.

        The client's IP address is not sent to the ReCaptcha server; instead,
        a completely random IP is generated and sent instead.

        :type request: :api:`twisted.web.http.Request`
        :param request: A ``Request`` object, including POST arguments which
            should include two key/value pairs: one key being
            ``'captcha_challenge_field'``, and the other,
            ``'captcha_response_field'``. These POST arguments should be
            obtained from :meth:`render_GET`.
        :rtupe: bool
        :returns: True, if the CAPTCHA solution was valid; False otherwise.
        """
        challenge, solution = self.extractClientSolution(request)
        clientIP = self.getClientIP(request)
        clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP)
        valid = captcha.GimpCaptcha.check(challenge, solution,
                                          self.secretKey, clientHMACKey)
        logging.debug("%sorrect captcha from %r: %r." % (
            "C" if valid else "Inc", Util.logSafely(clientIP), solution))

        return valid
Exemple #4
0
        def checkResponse(solution, request):
            """Check the :class:`txrecaptcha.RecaptchaResponse`.

            :type solution: :class:`txrecaptcha.RecaptchaResponse`.
            :param solution: The client's CAPTCHA solution, after it has been
                submitted to the reCaptcha API server.
            """
            # This valid CAPTCHA result from this function cannot be reliably
            # unittested, because it's callbacked to from the deferred
            # returned by ``txrecaptcha.submit``, the latter of which would
            # require networking (as well as automated CAPTCHA
            # breaking). Hence, the 'no cover' pragma.
            if solution.is_valid:  # pragma: no cover
                logging.info("Valid CAPTCHA solution from %r."
                             % Util.logSafely(clientIP))
                return (True, request)
            else:
                logging.info("Invalid CAPTCHA solution from %r: %r"
                             % (Util.logSafely(clientIP), solution.error_code))
                return (False, request)
Exemple #5
0
    def render_POST(self, request):
        try:
            challenge = request.args['recaptcha_challenge_field'][0]
            response = request.args['recaptcha_response_field'][0]
        except:
            return redirectTo(request.URLPath(), request)

        # generate a random IP for the captcha submission
        remote_ip = '%d.%d.%d.%d' % (randint(1,255),randint(1,255),
                                     randint(1,255),randint(1,255))

        recaptcha_response = captcha.submit(challenge, response,
                                        self.recaptchaPrivKey, remote_ip)
        if recaptcha_response.is_valid:
            logging.info("Valid recaptcha from %s. Parameters were %r",
                    Util.logSafely(remote_ip), request.args)
            return self.resource.render(request)
        else:
            logging.info("Invalid recaptcha from %s. Parameters were %r",
                         Util.logSafely(remote_ip), request.args)
            logging.info("Recaptcha error code: %s", recaptcha_response.error_code)
        return redirectTo(request.URLPath(), request)
Exemple #6
0
    def checkSolution(self, request):
        """Process a solved CAPTCHA by sending it to the ReCaptcha server.

        The client's IP address is not sent to the ReCaptcha server; instead,
        a completely random IP is generated and sent instead.

        :type request: :api:`twisted.web.http.Request`
        :param request: A ``Request`` object, including POST arguments which
            should include two key/value pairs: one key being
            ``'captcha_challenge_field'``, and the other,
            ``'captcha_response_field'``. These POST arguments should be
            obtained from :meth:`render_GET`.
        :rtupe: :api:`twisted.internet.defer.Deferred`
        :returns: the returned deferred will callback with a tuple of
                (``bool``, :api:`twisted.web.server.Request`). If the CAPTCHA
            solution was valid, a tuple will contain ``(True, request)``;
            otherwise, it will contain ``(False, request)``.
        """
        challenge, response = self.extractClientSolution(request)
        clientIP = self.getClientIP(request)
        remoteIP = self.getRemoteIP()

        logging.debug("Captcha from %r. Parameters: %r"
                      % (Util.logSafely(clientIP), request.args))

        def checkResponse(solution, request):
            """Check the :class:`txrecaptcha.RecaptchaResponse`.

            :type solution: :class:`txrecaptcha.RecaptchaResponse`.
            :param solution: The client's CAPTCHA solution, after it has been
                submitted to the reCaptcha API server.
            """
            # This valid CAPTCHA result from this function cannot be reliably
            # unittested, because it's callbacked to from the deferred
            # returned by ``txrecaptcha.submit``, the latter of which would
            # require networking (as well as automated CAPTCHA
            # breaking). Hence, the 'no cover' pragma.
            if solution.is_valid:  # pragma: no cover
                logging.info("Valid CAPTCHA solution from %r."
                             % Util.logSafely(clientIP))
                return (True, request)
            else:
                logging.info("Invalid CAPTCHA solution from %r: %r"
                             % (Util.logSafely(clientIP), solution.error_code))
                return (False, request)

        d = txrecaptcha.submit(challenge, response, self.recaptchaPrivKey,
                               remoteIP).addCallback(checkResponse, request)
        return d
Exemple #7
0
def configureLogging(cfg):
    """Set up Python's logging subsystem based on the configuratino.
    """

    # Turn on safe logging by default
    safelogging = getattr(cfg, 'SAFELOGGING', True)

    level = getattr(cfg, 'LOGLEVEL', 'WARNING')
    level = getattr(logging, level)
    logfile = getattr(cfg, 'LOGFILE', "")
    logfile_count = getattr(cfg, 'LOGFILE_COUNT', 5)
    logfile_rotate_size = getattr(cfg, 'LOGFILE_ROTATE_SIZE', 10000000)
    Util.set_safe_logging(safelogging)

    logging.getLogger().setLevel(level)
    if logfile:
        handler = logging.handlers.RotatingFileHandler(logfile, 'a',
                                                       logfile_rotate_size,
                                                       logfile_count)
        formatter = logging.Formatter(
            '%(asctime)s [%(levelname)s] %(message)s', "%b %d %H:%M:%S")
        handler.setFormatter(formatter)
        logging.getLogger().addHandler(handler)

    logging.info("Logger Started.")
    logging.info("Level: %s", level)
    if logfile:
        logging.info("Log File: %s", os.path.abspath(logfile))
        logging.info("Log File Count: %d", logfile_count)
        logging.info("Rotate Logs After Size: %d", logfile_rotate_size)
    else:
        logging.info("Logging to stderr")
    if safelogging:
        logging.info("Safe Logging: Enabled")
    else:
        logging.warn("Safe Logging: Disabled")
Exemple #8
0
def replyToMail(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, possibly send a reply.
    """
    logging.info("Got a completed email; deciding whether to reply.")
    sendToUser, response = getMailResponse(lines, ctx)
    if response is None:
        logging.debug("getMailResponse said not to reply, so I won't.")
        return
    response.seek(0)
    d = Deferred()
    factory = twisted.mail.smtp.SMTPSenderFactory(
        ctx.smtpFromAddr,
        sendToUser,
        response,
        d)
    reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory)
    logging.info("Sending reply to %r", Util.logSafely(sendToUser))
    return d
Exemple #9
0
def composeEmail(fromAddr, clientAddr, subject, body, msgID=False,
        gpgContext=None):

    f = StringIO()
    w = MimeWriter.MimeWriter(f)
    w.addheader("From", fromAddr)
    w.addheader("To", clientAddr)
    w.addheader("Message-ID", twisted.mail.smtp.messageid())
    if not subject.startswith("Re:"): subject = "Re: %s"%subject
    w.addheader("Subject", subject)
    if msgID:
        w.addheader("In-Reply-To", msgID)
    w.addheader("Date", twisted.mail.smtp.rfc822date())
    mailbody = w.startbody("text/plain")

    # gpg-clearsign messages
    if gpgContext:
        signature = StringIO()
        plaintext = StringIO(body)
        sigs = gpgContext.sign(plaintext, signature, gpgme.SIG_MODE_CLEAR)
        if (len(sigs) != 1):
            logging.warn('Failed to sign message!')
        signature.seek(0)
        [mailbody.write(l) for l in signature]
    else:
        mailbody.write(body)

    # Only log the email text (including all headers) if SAFE_LOGGING is
    # disabled:
    if not Util.safe_logging:
        f.seek(0)
        logging.debug("Email contents:\n%s" % f.read())
    else:
        logging.debug("Email text for %r created." % Util.logSafely(clientAddr))
    f.seek(0)

    return clientAddr, f
Exemple #10
0
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)

    except IgnoreEmail, e:
        logging.info("Got a mail too frequently; ignoring %r: %s.",
                      Util.logSafely(clientAddr), e)
        return None, None 

    except BadEmail, e:
        logging.info("Got a mail from a bad email address %r: %s.",
                     Util.logSafely(clientAddr), e)
        return None, None 

    if bridges:
        with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS
        answer = "".join("  %s\n" %b.getConfigLine(
            includeFingerprint=with_fp,
            addressClass=addressClass,
            transport=transport,
            request=clientAddr
            ) for b in bridges)
Exemple #11
0
    def getBridgesForEmail(self,
                           emailaddress,
                           epoch,
                           N=1,
                           parameters=None,
                           countryCode=None,
                           bridgeFilterRules=None):
        """Return a list of bridges to give to a user.
           emailaddress -- the user's email address, as given in a from line.
           epoch -- the time period when we got this request.  This can
               be any string, so long as it changes with every period.
           N -- the number of bridges to try to give back.
        """
        if not bridgeFilterRules:
            bridgeFilterRules = []
        now = time.time()
        try:
            emailaddress = normalizeEmail(emailaddress, self.domainmap,
                                          self.domainrules)
        except BadEmail:
            return []  #XXXX log the exception
        if emailaddress is None:
            return []  #XXXX raise an exception.

        db = bridgedb.Storage.getDB()
        wasWarned = db.getWarnedEmail(emailaddress)

        lastSaw = db.getEmailTime(emailaddress)
        if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now:
            if wasWarned:
                logging.info(
                    "Got a request for bridges from %r; we already "
                    "sent a warning. Ignoring.", Util.logSafely(emailaddress))
                raise IgnoreEmail("Client was warned",
                                  Util.logSafely(emailaddress))
            else:
                db.setWarnedEmail(emailaddress, True, now)
                db.commit()

            logging.info(
                "Got a request for bridges from %r; we already "
                "answered one within the last %d seconds. Warning.",
                Util.logSafely(emailaddress), MAX_EMAIL_RATE)
            raise TooSoonEmail("Too many emails; wait till later",
                               emailaddress)

        # warning period is over
        elif wasWarned:
            db.setWarnedEmail(emailaddress, False)

        pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))

        ring = None
        ruleset = frozenset(bridgeFilterRules)
        if ruleset in self.splitter.filterRings.keys():
            logging.debug("Cache hit %s" % ruleset)
            _, ring = self.splitter.filterRings[ruleset]
        else:
            # cache miss, add new ring
            logging.debug("Cache miss %s" % ruleset)

            # add new ring
            key1 = bridgedb.Bridges.get_hmac(self.splitter.key,
                                             "Order-Bridges-In-Ring")
            ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters)
            # debug log: cache miss
            self.splitter.addRing(ring,
                                  ruleset,
                                  filterBridgesByRules(bridgeFilterRules),
                                  populate_from=self.splitter.bridges)

        result = ring.getBridges(
            pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))

        db.setEmailTime(emailaddress, now)
        db.commit()
        return result
Exemple #12
0
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)

    except IgnoreEmail, e:
        logging.info("Got a mail too frequently; ignoring %r: %s.",
                      Util.logSafely(clientAddr), e)
        return None, None 

    except BadEmail, e:
        logging.info("Got a mail from a bad email address %r: %s.",
                     Util.logSafely(clientAddr), e)
        return None, None 

    if bridges:
        with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS
        answer = "".join("  %s\n" %b.getConfigLine(
            includeFingerprint=with_fp,
            addressClass=addressClass,
            transport=transport,
            request=clientAddr
            ) for b in bridges)
Exemple #13
0
    def getBridgeRequestAnswer(self, request):
        """ returns a response to a bridge request """
        interval = self.schedule.getInterval(time.time())
        bridges = ( )
        ip = None
        countryCode = None
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        if geoip:
            countryCode = geoip.country_code_by_addr(ip)

        # set locale
        setLocaleFromRequestHeader(request)

        format = request.args.get("format", None)
        if format and len(format): format = format[0] # choose the first arg

        # do want any options?
        transport = ipv6 = unblocked = False

        ipv6 = request.args.get("ipv6", False)
        if ipv6: ipv6 = True # if anything after ?ipv6=

        try:
            # validate method name
            transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
                    request.args.get("transport")[0]).group()
        except (TypeError, IndexError, AttributeError):
            transport = None

        try:
            unblocked = re.match('[a-zA-Z]{2,4}',
                    request.args.get("unblocked")[0]).group()
        except (TypeError, IndexError, AttributeError):
            unblocked = False

        rules = []

        if ip:
            if ipv6:
                rules.append(filterBridgesByIP6)
                addressClass = IPv6Address
            else:
                rules.append(filterBridgesByIP4)
                addressClass = IPv4Address

            if transport:
                #XXX: A cleaner solution would differentiate between
                # addresses by protocol rather than have separate lists
                # Tor to be a transport, and selecting between them.
                rules = [filterBridgesByTransport(transport, addressClass)]

            if unblocked:
                rules.append(filterBridgesByNotBlockedIn(unblocked,
                    addressClass, transport))

            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive,
                                                       countryCode,
                                                       bridgeFilterRules=rules)

        answer = None
        if bridges:
            answer = "".join("  %s\n" % b.getConfigLine(
                includeFingerprint=self.includeFingerprints,
                addressClass=addressClass,
                transport=transport,
                request=bridgedb.Dist.uniformMap(ip)
                ) for b in bridges) 

        logging.info("Replying to web request from %s.  Parameters were %r",
                     Util.logSafely(ip), request.args)
        if format == 'plain':
            request.setHeader("Content-Type", "text/plain")
            return answer
        else:
            request.setHeader("Content-Type", "text/html; charset=utf-8")
            return lookup.get_template('bridges.html').render(answer=answer)
Exemple #14
0
def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    # RFC822 requires at least one 'To' address
    clientToList = msg.getaddrlist("To")
    clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
    msgID = msg.getheader("Message-ID", None)
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    # Look up the locale part in the 'To:' address, if there is one and get
    # the appropriate Translation object
    lang = getLocaleFromPlusAddr(clientToaddr)
    t = I18n.getLang(lang)

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except BadEmail:
        logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
        logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
        return None,None
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    #for ln in lines:
    #    if ln.strip().lower() in ("get bridges", "subject: get bridges"):
    #        break
    #else:
    #    logging.info("Got a mail from %r with no bridge request; dropping",
    #                 clientAddr)
    #    return None,None

    # Figure out which bridges to send
    unblocked = transport = ipv6 = skippedheaders = False
    bridgeFilterRules = []
    addressClass = None
    for ln in lines:
        # ignore all lines before the subject header
        if "subject" in ln.strip().lower():
            skippedheaders = True
        if not skippedheaders: continue

        if "ipv6" in ln.strip().lower():
            ipv6 = True
        if "transport" in ln.strip().lower():
            try:
                transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None
            logging.debug("Got request for transport: %s" % transport)
        if "unblocked" in ln.strip().lower():
            try:
                unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None

    if ipv6:
        bridgeFilterRules.append(filterBridgesByIP6)
        addressClass = IPv6Address
    else:
        bridgeFilterRules.append(filterBridgesByIP4)
        addressClass = IPv4Address

    if transport:
        bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]

    if unblocked:
        rules.append(filterBridgesByNotBlockedIn(unblocked,
            addressClass, transport))

    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
            interval, ctx.N,
            countryCode=None,
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)
Exemple #15
0
    def getBridgesForIP(self,
                        ip,
                        epoch,
                        N=1,
                        countryCode=None,
                        bridgeFilterRules=None):
        """Return a list of bridges to give to a user.
           ip -- the user's IP address, as a dotted quad.
           epoch -- the time period when we got this request.  This can
               be any string, so long as it changes with every period.
           N -- the number of bridges to try to give back.
        """
        if not bridgeFilterRules: bridgeFilterRules = []
        logging.debug("getBridgesForIP(%s, %s, %s, %s", Util.logSafely(ip),
                      epoch, N, bridgeFilterRules)
        if not len(self.splitter):
            logging.debug("bailing without splitter")
            return []

        area = self.areaMapper(ip)

        logging.info("area is %s", Util.logSafely(area))

        key1 = ''
        pos = 0
        n = self.nClusters

        # only one of ip categories or area clustering is active
        # try to match the request to an ip category
        for category in self.categories:
            # IP Categories
            if category.contains(ip):
                g = filterAssignBridgesToRing(
                    self.splitter.hmac, self.nClusters + len(self.categories),
                    n)
                bridgeFilterRules.append(g)
                logging.info("category<%s>%s", epoch, Util.logSafely(area))
                pos = self.areaOrderHmac("category<%s>%s" % (epoch, area))
                key1 = bridgedb.Bridges.get_hmac(
                    self.splitter.key, "Order-Bridges-In-Ring-%d" % n)
                break
            n += 1

        # if no category matches, use area clustering
        else:
            # IP clustering
            h = int(self.areaClusterHmac(area)[:8], 16)
            # length of numClusters
            clusterNum = h % self.nClusters

            g = filterAssignBridgesToRing(
                self.splitter.hmac, self.nClusters + len(self.categories),
                clusterNum)
            bridgeFilterRules.append(g)
            pos = self.areaOrderHmac("<%s>%s" % (epoch, area))
            key1 = bridgedb.Bridges.get_hmac(
                self.splitter.key, "Order-Bridges-In-Ring-%d" % clusterNum)

        logging.debug("bridgeFilterRules: %s" % bridgeFilterRules)

        # try to find a cached copy
        ruleset = frozenset(bridgeFilterRules)

        # See if we have a cached copy of the ring,
        # otherwise, add a new ring and populate it
        if ruleset in self.splitter.filterRings.keys():
            logging.debug("Cache hit %s" % ruleset)
            _, ring = self.splitter.filterRings[ruleset]

        # else create the ring and populate it
        else:
            logging.debug("Cache miss %s" % ruleset)
            ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters)
            self.splitter.addRing(ring,
                                  ruleset,
                                  filterBridgesByRules(bridgeFilterRules),
                                  populate_from=self.splitter.bridges)

        # get an appropriate number of bridges
        return ring.getBridges(
            pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))
Exemple #16
0
    def getBridgesForEmail(self, emailaddress, epoch, N=1,
            parameters=None, countryCode=None, bridgeFilterRules=None):
        """Return a list of bridges to give to a user.
           emailaddress -- the user's email address, as given in a from line.
           epoch -- the time period when we got this request.  This can
               be any string, so long as it changes with every period.
           N -- the number of bridges to try to give back.
        """
        if not bridgeFilterRules:
            bridgeFilterRules=[]
        now = time.time()
        try:
            emailaddress = normalizeEmail(emailaddress, self.domainmap,
                                          self.domainrules)
        except BadEmail:
            return [] #XXXX log the exception
        if emailaddress is None:
            return [] #XXXX raise an exception.

        db = bridgedb.Storage.getDB()
        wasWarned = db.getWarnedEmail(emailaddress)

        lastSaw = db.getEmailTime(emailaddress)
        if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now:
            if wasWarned:
                logging.info("Got a request for bridges from %r; we already "
                             "sent a warning. Ignoring.", Util.logSafely(emailaddress))
                raise IgnoreEmail("Client was warned", Util.logSafely(emailaddress))
            else:
                db.setWarnedEmail(emailaddress, True, now)
                db.commit() 

            logging.info("Got a request for bridges from %r; we already "
                         "answered one within the last %d seconds. Warning.",
                         Util.logSafely(emailaddress), MAX_EMAIL_RATE)
            raise TooSoonEmail("Too many emails; wait till later", emailaddress)

        # warning period is over
        elif wasWarned:
            db.setWarnedEmail(emailaddress, False) 

        pos = self.emailHmac("<%s>%s" % (epoch, emailaddress))

        ring = None
        ruleset = frozenset(bridgeFilterRules)
        if ruleset in self.splitter.filterRings.keys():
            logging.debug("Cache hit %s" % ruleset)
            _,ring = self.splitter.filterRings[ruleset]
        else:
            # cache miss, add new ring
            logging.debug("Cache miss %s" % ruleset)

            # add new ring 
            key1 = bridgedb.Bridges.get_hmac(self.splitter.key,
                    "Order-Bridges-In-Ring")
            ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters)
            # debug log: cache miss 
            self.splitter.addRing(ring, ruleset,
                                  filterBridgesByRules(bridgeFilterRules),
                                  populate_from=self.splitter.bridges)

        result = ring.getBridges(pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))

        db.setEmailTime(emailaddress, now)
        db.commit()
        return result
Exemple #17
0
    def getBridgesForIP(self, ip, epoch, N=1, countryCode=None,
                        bridgeFilterRules=None):
        """Return a list of bridges to give to a user.
           ip -- the user's IP address, as a dotted quad.
           epoch -- the time period when we got this request.  This can
               be any string, so long as it changes with every period.
           N -- the number of bridges to try to give back.
        """
        if not bridgeFilterRules: bridgeFilterRules=[]
        logging.debug("getBridgesForIP(%s, %s, %s, %s",
                Util.logSafely(ip), epoch, N, bridgeFilterRules)
        if not len(self.splitter):
            logging.debug("bailing without splitter")
            return []

        area = self.areaMapper(ip)

        logging.info("area is %s", Util.logSafely(area))
        
        key1 = ''
        pos = 0
        n = self.nClusters

        # only one of ip categories or area clustering is active
        # try to match the request to an ip category
        for category in self.categories:
            # IP Categories
            if category.contains(ip):
                g = filterAssignBridgesToRing(self.splitter.hmac,
                                                      self.nClusters +
                                                      len(self.categories),
                                                      n)
                bridgeFilterRules.append(g)
                logging.info("category<%s>%s", epoch, Util.logSafely(area))
                pos = self.areaOrderHmac("category<%s>%s" % (epoch, area))
                key1 = bridgedb.Bridges.get_hmac(self.splitter.key,
                                             "Order-Bridges-In-Ring-%d"%n) 
                break;
            n += 1

        # if no category matches, use area clustering
        else:
            # IP clustering
            h = int( self.areaClusterHmac(area)[:8], 16)
            # length of numClusters
            clusterNum = h % self.nClusters
 
            g = filterAssignBridgesToRing(self.splitter.hmac,
                                          self.nClusters +
                                          len(self.categories),
                                          clusterNum) 
            bridgeFilterRules.append(g)
            pos = self.areaOrderHmac("<%s>%s" % (epoch, area))
            key1 = bridgedb.Bridges.get_hmac(self.splitter.key,
                                             "Order-Bridges-In-Ring-%d"%clusterNum) 

        logging.debug("bridgeFilterRules: %s" % bridgeFilterRules)

        # try to find a cached copy
        ruleset = frozenset(bridgeFilterRules)

        # See if we have a cached copy of the ring,
        # otherwise, add a new ring and populate it
        if ruleset in self.splitter.filterRings.keys():
            logging.debug("Cache hit %s" % ruleset)
            _,ring = self.splitter.filterRings[ruleset]

        # else create the ring and populate it
        else:
            logging.debug("Cache miss %s" % ruleset)
            ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters)
            self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules),
                                  populate_from=self.splitter.bridges)

        # get an appropriate number of bridges
        return ring.getBridges(pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))
Exemple #18
0
def getMailResponse(lines, ctx):
    """Given a list of lines from an incoming email message, and a
       MailContext object, parse the email and decide what to do in response.
       If we want to answer, return a 2-tuple containing the address that
       will receive the response, and a readable filelike object containing
       the response.  Return None,None if we shouldn't answer.
    """
    # Extract data from the headers.
    msg = rfc822.Message(MailFile(lines))
    subject = msg.getheader("Subject", None)
    if not subject: subject = "[no subject]"
    clientFromAddr = msg.getaddr("From")
    clientSenderAddr = msg.getaddr("Sender")
    # RFC822 requires at least one 'To' address
    clientToList = msg.getaddrlist("To")
    clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList)
    msgID = msg.getheader("Message-ID", None)
    if clientSenderAddr and clientSenderAddr[1]:
        clientAddr = clientSenderAddr[1]
    elif clientFromAddr and clientFromAddr[1]:
        clientAddr = clientFromAddr[1]
    else:
        logging.info("No From or Sender header on incoming mail.")
        return None,None

    # Look up the locale part in the 'To:' address, if there is one and get
    # the appropriate Translation object
    lang = getLocaleFromPlusAddr(clientToaddr)
    t = I18n.getLang(lang)

    try:
        _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower())
    except BadEmail:
        logging.info("Ignoring bad address on incoming email.")
        return None,None
    if not addrdomain:
        logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr))
    if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP:
        addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain)
    if addrdomain not in ctx.cfg.EMAIL_DOMAINS:
        logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain))
        return None,None
    rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, [])
    if 'dkim' in rules:
        # getheader() returns the last of a given kind of header; we want
        # to get the first, so we use getheaders() instead.
        dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results")
        dkimHeader = "<no header>"
        if dkimHeaders: dkimHeader = dkimHeaders[0]
        if not dkimHeader.startswith("pass"):
            logging.info("Got a bad dkim header (%r) on an incoming mail; "
                         "rejecting it.", dkimHeader)
            return None, None

    # Was the magic string included
    #for ln in lines:
    #    if ln.strip().lower() in ("get bridges", "subject: get bridges"):
    #        break
    #else:
    #    logging.info("Got a mail from %r with no bridge request; dropping",
    #                 clientAddr)
    #    return None,None

    # Figure out which bridges to send
    unblocked = transport = ipv6 = skippedheaders = False
    bridgeFilterRules = []
    addressClass = None
    for ln in lines:
        # ignore all lines before the subject header
        if "subject" in ln.strip().lower():
            skippedheaders = True
        if not skippedheaders: continue

        if "ipv6" in ln.strip().lower():
            ipv6 = True
        if "transport" in ln.strip().lower():
            try:
                transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None
            logging.debug("Got request for transport: %s" % transport)
        if "unblocked" in ln.strip().lower():
            try:
                unblocked = re.search("unblocked ([a-zA-Z]{2,4})",
                        ln).group(1).strip()
            except (TypeError, AttributeError):
                transport = None

    if ipv6:
        bridgeFilterRules.append(filterBridgesByIP6)
        addressClass = IPv6Address
    else:
        bridgeFilterRules.append(filterBridgesByIP4)
        addressClass = IPv4Address

    if transport:
        bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)]

    if unblocked:
        rules.append(filterBridgesByNotBlockedIn(unblocked,
            addressClass, transport))

    try:
        interval = ctx.schedule.getInterval(time.time())
        bridges = ctx.distributor.getBridgesForEmail(clientAddr,
            interval, ctx.N,
            countryCode=None,
            bridgeFilterRules=bridgeFilterRules)

    # Handle rate limited email
    except TooSoonEmail, e:
        logging.info("Got a mail too frequently; warning %r: %s.",
                     Util.logSafely(clientAddr), e)

        # Compose a warning email
        # MAX_EMAIL_RATE is in seconds, convert to hours
        body  = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600)
        return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID,
                gpgContext=ctx.gpgContext)
Exemple #19
0
    def getBridgeRequestAnswer(self, request):
        """Respond to a client HTTP request for bridges.

        :type request: :api:`twisted.web.http.Request`
        :param request: A ``Request`` object containing the HTTP method, full
            URI, and any URL/POST arguments and headers present.
        :rtype: str
        :returns: A plaintext or HTML response to serve.
        """
        # XXX why are we getting the interval if our distributor might be
        # using bridgedb.Time.NoSchedule?
        interval = self.schedule.getInterval(time.time())
        bridges = ( )
        ip = None
        countryCode = None

        # XXX this code is duplicated in CaptchaProtectedResource
        if self.useForwardedHeader:
            h = request.getHeader("X-Forwarded-For")
            if h:
                ip = h.split(",")[-1].strip()
                if not bridgedb.Bridges.is_valid_ip(ip):
                    logging.warn("Got weird forwarded-for value %r",h)
                    ip = None
        else:
            ip = request.getClientIP()

        # XXX This can also be a separate function
        # XXX if the ip is None, this throws an exception
        if geoip:
            countryCode = geoip.country_code_by_addr(ip)
            if countryCode:
                logging.debug("Client request from GeoIP CC: %s" % countryCode)

        rtl = usingRTLLang(request)
        if rtl:
            logging.debug("Rendering RTL response.")

        # XXX separate function again
        format = request.args.get("format", None)
        if format and len(format): format = format[0] # choose the first arg

        # do want any options?
        transport = ipv6 = unblocked = False

        ipv6 = request.args.get("ipv6", False)
        if ipv6: ipv6 = True # if anything after ?ipv6=

        # XXX oh dear hell. why not check for the '?transport=' arg before
        # regex'ing? And why not compile the regex once, somewhere outside
        # this function and class?
        try:
            # validate method name
            transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*',
                    request.args.get("transport")[0]).group()
        except (TypeError, IndexError, AttributeError):
            transport = None

        try:
            unblocked = re.match('[a-zA-Z]{2,4}',
                    request.args.get("unblocked")[0]).group()
        except (TypeError, IndexError, AttributeError):
            unblocked = False

        logging.info("Replying to web request from %s. Parameters were %r"
                     % (Util.logSafely(ip), request.args))

        rules = []
        bridgeLines = None

        if ip:
            if ipv6:
                rules.append(filterBridgesByIP6)
                addressClass = IPv6Address
            else:
                rules.append(filterBridgesByIP4)
                addressClass = IPv4Address

            if transport:
                #XXX: A cleaner solution would differentiate between
                # addresses by protocol rather than have separate lists
                # Tor to be a transport, and selecting between them.
                rules = [filterBridgesByTransport(transport, addressClass)]

            if unblocked:
                rules.append(filterBridgesByNotBlockedIn(unblocked,
                    addressClass, transport))

            bridges = self.distributor.getBridgesForIP(ip, interval,
                                                       self.nBridgesToGive,
                                                       countryCode,
                                                       bridgeFilterRules=rules)
            bridgeLines = "".join("  %s\n" % b.getConfigLine(
                includeFingerprint=self.includeFingerprints,
                addressClass=addressClass,
                transport=transport,
                request=bridgedb.Dist.uniformMap(ip)
                ) for b in bridges)

        answer = self.renderAnswer(request, bridgeLines, rtl, format)
        return answer
Exemple #20
0
    def getBridgesForIP(self, ip, epoch, N=1, countryCode=None,
                        bridgeFilterRules=None):
        """Return a list of bridges to give to a user.

        :param str ip: The user's IP address, as a dotted quad.
        :param str epoch: The time period when we got this request.  This can
                          be any string, so long as it changes with every
                          period.
        :param int N: The number of bridges to try to give back. (default: 1)
        :param str countryCode: DOCDOC (default: None)
        :param list bridgeFilterRules: A list of callables used filter the
                                       bridges returned in the response to the
                                       client. See :mod:`~bridgedb.Filters`.
        :rtype: list
        :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in
                 the response. See
                 :meth:`bridgedb.HTTPServer.WebResource.getBridgeRequestAnswer`
                 for an example of how this is used.
        """
        logging.info("Attempting to return %d bridges to client %s..."
                     % (N, Util.logSafely(ip)))

        if not bridgeFilterRules:
            bridgeFilterRules=[]

        if not len(self.splitter):
            logging.warn("Bailing! Splitter has zero bridges!")
            return []

        logging.debug("Bridges in splitter:\t%d" % len(self.splitter))
        logging.debug("Client request epoch:\t%s" % epoch)
        logging.debug("Active bridge filters:\t%s"
                      % ' '.join([x.func_name for x in bridgeFilterRules]))

        area = self.areaMapper(ip)
        logging.debug("IP mapped to area:\t%s"
                      % Util.logSafely("{0}.0/24".format(area)))

        key1 = ''
        pos = 0
        n = self.nClusters

        # only one of ip categories or area clustering is active
        # try to match the request to an ip category
        for category in self.categories:
            # IP Categories
            if category.contains(ip):
                g = filterAssignBridgesToRing(self.splitter.hmac,
                                              self.nClusters +
                                              len(self.categories),
                                              n)
                bridgeFilterRules.append(g)
                logging.info("category<%s>%s", epoch, Util.logSafely(area))
                pos = self.areaOrderHmac("category<%s>%s" % (epoch, area))
                key1 = getHMAC(self.splitter.key,
                               "Order-Bridges-In-Ring-%d" % n)
                break
            n += 1

        # if no category matches, use area clustering
        else:
            # IP clustering
            h = int( self.areaClusterHmac(area)[:8], 16)
            # length of numClusters
            clusterNum = h % self.nClusters

            g = filterAssignBridgesToRing(self.splitter.hmac,
                                          self.nClusters +
                                          len(self.categories),
                                          clusterNum)
            bridgeFilterRules.append(g)
            pos = self.areaOrderHmac("<%s>%s" % (epoch, area))
            key1 = getHMAC(self.splitter.key,
                           "Order-Bridges-In-Ring-%d" % clusterNum)

        # try to find a cached copy
        ruleset = frozenset(bridgeFilterRules)

        # See if we have a cached copy of the ring,
        # otherwise, add a new ring and populate it
        if ruleset in self.splitter.filterRings.keys():
            logging.debug("Cache hit %s" % ruleset)
            _,ring = self.splitter.filterRings[ruleset]

        # else create the ring and populate it
        else:
            logging.debug("Cache miss %s" % ruleset)
            ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters)
            self.splitter.addRing(ring,
                                  ruleset,
                                  filterBridgesByRules(bridgeFilterRules),
                                  populate_from=self.splitter.bridges)

        # get an appropriate number of bridges
        numBridgesToReturn = getNumBridgesPerAnswer(ring,
                                                    max_bridges_per_answer=N)
        answer = ring.getBridges(pos, numBridgesToReturn)
        return answer