def configureLogging(cfg): """Set up Python's logging subsystem based on the configuratino. """ # Turn on safe logging by default safelogging = getattr(cfg, "SAFELOGGING", True) level = getattr(cfg, "LOGLEVEL", "WARNING") level = getattr(logging, level) logfile = getattr(cfg, "LOGFILE", "") logfile_count = getattr(cfg, "LOGFILE_COUNT", 5) logfile_rotate_size = getattr(cfg, "LOGFILE_ROTATE_SIZE", 10000000) Util.set_safe_logging(safelogging) logging.getLogger().setLevel(level) if logfile: handler = logging.handlers.RotatingFileHandler(logfile, "a", logfile_rotate_size, logfile_count) formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s", "%b %d %H:%M:%S") handler.setFormatter(formatter) logging.getLogger().addHandler(handler) logging.info("Logger Started.") logging.info("Level: %s", level) if logfile: logging.info("Log File: %s", os.path.abspath(logfile)) logging.info("Log File Count: %d", logfile_count) logging.info("Rotate Logs After Size: %d", logfile_rotate_size) else: logging.info("Logging to stderr") if safelogging: logging.info("Safe Logging: Enabled") else: logging.warn("Safe Logging: Disabled")
def replyToMail(lines, ctx): """Reply to an incoming email. Maybe. If no `response` is returned from :func:`getMailResponse`, then the incoming email will not be responded to at all. This can happen for several reasons, for example: if the DKIM signature was invalid or missing, or if the incoming email came from an unacceptable domain, or if there have been too many emails from this client in the allotted time period. :param list lines: A list of lines from an incoming email message. :type ctx: :class:`MailContext` :param ctx: The configured context for the email server. """ logging.info("Got an email; deciding whether to reply.") sendToUser, response = getMailResponse(lines, ctx) if response is None: logging.debug("getMailResponse() said not to reply to %s, so I won't." % Util.logSafely(sendToUser)) return response.seek(0) logging.info("Sending reply to %r", Util.logSafely(sendToUser)) d = Deferred() factory = twisted.mail.smtp.SMTPSenderFactory(ctx.smtpFromAddr, sendToUser, response, d) reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) return d
def checkSolution(self, request): """Process a solved CAPTCHA by sending rehashing the solution together with the client's IP address, and checking that the result matches the challenge. The client's IP address is not sent to the ReCaptcha server; instead, a completely random IP is generated and sent instead. :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object, including POST arguments which should include two key/value pairs: one key being ``'captcha_challenge_field'``, and the other, ``'captcha_response_field'``. These POST arguments should be obtained from :meth:`render_GET`. :rtupe: bool :returns: True, if the CAPTCHA solution was valid; False otherwise. """ challenge, solution = self.extractClientSolution(request) clientIP = self.getClientIP(request) clientHMACKey = crypto.getHMAC(self.hmacKey, clientIP) valid = captcha.GimpCaptcha.check(challenge, solution, self.secretKey, clientHMACKey) logging.debug("%sorrect captcha from %r: %r." % ( "C" if valid else "Inc", Util.logSafely(clientIP), solution)) return valid
def checkResponse(solution, request): """Check the :class:`txrecaptcha.RecaptchaResponse`. :type solution: :class:`txrecaptcha.RecaptchaResponse`. :param solution: The client's CAPTCHA solution, after it has been submitted to the reCaptcha API server. """ # This valid CAPTCHA result from this function cannot be reliably # unittested, because it's callbacked to from the deferred # returned by ``txrecaptcha.submit``, the latter of which would # require networking (as well as automated CAPTCHA # breaking). Hence, the 'no cover' pragma. if solution.is_valid: # pragma: no cover logging.info("Valid CAPTCHA solution from %r." % Util.logSafely(clientIP)) return (True, request) else: logging.info("Invalid CAPTCHA solution from %r: %r" % (Util.logSafely(clientIP), solution.error_code)) return (False, request)
def render_POST(self, request): try: challenge = request.args['recaptcha_challenge_field'][0] response = request.args['recaptcha_response_field'][0] except: return redirectTo(request.URLPath(), request) # generate a random IP for the captcha submission remote_ip = '%d.%d.%d.%d' % (randint(1,255),randint(1,255), randint(1,255),randint(1,255)) recaptcha_response = captcha.submit(challenge, response, self.recaptchaPrivKey, remote_ip) if recaptcha_response.is_valid: logging.info("Valid recaptcha from %s. Parameters were %r", Util.logSafely(remote_ip), request.args) return self.resource.render(request) else: logging.info("Invalid recaptcha from %s. Parameters were %r", Util.logSafely(remote_ip), request.args) logging.info("Recaptcha error code: %s", recaptcha_response.error_code) return redirectTo(request.URLPath(), request)
def checkSolution(self, request): """Process a solved CAPTCHA by sending it to the ReCaptcha server. The client's IP address is not sent to the ReCaptcha server; instead, a completely random IP is generated and sent instead. :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object, including POST arguments which should include two key/value pairs: one key being ``'captcha_challenge_field'``, and the other, ``'captcha_response_field'``. These POST arguments should be obtained from :meth:`render_GET`. :rtupe: :api:`twisted.internet.defer.Deferred` :returns: the returned deferred will callback with a tuple of (``bool``, :api:`twisted.web.server.Request`). If the CAPTCHA solution was valid, a tuple will contain ``(True, request)``; otherwise, it will contain ``(False, request)``. """ challenge, response = self.extractClientSolution(request) clientIP = self.getClientIP(request) remoteIP = self.getRemoteIP() logging.debug("Captcha from %r. Parameters: %r" % (Util.logSafely(clientIP), request.args)) def checkResponse(solution, request): """Check the :class:`txrecaptcha.RecaptchaResponse`. :type solution: :class:`txrecaptcha.RecaptchaResponse`. :param solution: The client's CAPTCHA solution, after it has been submitted to the reCaptcha API server. """ # This valid CAPTCHA result from this function cannot be reliably # unittested, because it's callbacked to from the deferred # returned by ``txrecaptcha.submit``, the latter of which would # require networking (as well as automated CAPTCHA # breaking). Hence, the 'no cover' pragma. if solution.is_valid: # pragma: no cover logging.info("Valid CAPTCHA solution from %r." % Util.logSafely(clientIP)) return (True, request) else: logging.info("Invalid CAPTCHA solution from %r: %r" % (Util.logSafely(clientIP), solution.error_code)) return (False, request) d = txrecaptcha.submit(challenge, response, self.recaptchaPrivKey, remoteIP).addCallback(checkResponse, request) return d
def configureLogging(cfg): """Set up Python's logging subsystem based on the configuratino. """ # Turn on safe logging by default safelogging = getattr(cfg, 'SAFELOGGING', True) level = getattr(cfg, 'LOGLEVEL', 'WARNING') level = getattr(logging, level) logfile = getattr(cfg, 'LOGFILE', "") logfile_count = getattr(cfg, 'LOGFILE_COUNT', 5) logfile_rotate_size = getattr(cfg, 'LOGFILE_ROTATE_SIZE', 10000000) Util.set_safe_logging(safelogging) logging.getLogger().setLevel(level) if logfile: handler = logging.handlers.RotatingFileHandler(logfile, 'a', logfile_rotate_size, logfile_count) formatter = logging.Formatter( '%(asctime)s [%(levelname)s] %(message)s', "%b %d %H:%M:%S") handler.setFormatter(formatter) logging.getLogger().addHandler(handler) logging.info("Logger Started.") logging.info("Level: %s", level) if logfile: logging.info("Log File: %s", os.path.abspath(logfile)) logging.info("Log File Count: %d", logfile_count) logging.info("Rotate Logs After Size: %d", logfile_rotate_size) else: logging.info("Logging to stderr") if safelogging: logging.info("Safe Logging: Enabled") else: logging.warn("Safe Logging: Disabled")
def replyToMail(lines, ctx): """Given a list of lines from an incoming email message, and a MailContext object, possibly send a reply. """ logging.info("Got a completed email; deciding whether to reply.") sendToUser, response = getMailResponse(lines, ctx) if response is None: logging.debug("getMailResponse said not to reply, so I won't.") return response.seek(0) d = Deferred() factory = twisted.mail.smtp.SMTPSenderFactory( ctx.smtpFromAddr, sendToUser, response, d) reactor.connectTCP(ctx.smtpServer, ctx.smtpPort, factory) logging.info("Sending reply to %r", Util.logSafely(sendToUser)) return d
def composeEmail(fromAddr, clientAddr, subject, body, msgID=False, gpgContext=None): f = StringIO() w = MimeWriter.MimeWriter(f) w.addheader("From", fromAddr) w.addheader("To", clientAddr) w.addheader("Message-ID", twisted.mail.smtp.messageid()) if not subject.startswith("Re:"): subject = "Re: %s"%subject w.addheader("Subject", subject) if msgID: w.addheader("In-Reply-To", msgID) w.addheader("Date", twisted.mail.smtp.rfc822date()) mailbody = w.startbody("text/plain") # gpg-clearsign messages if gpgContext: signature = StringIO() plaintext = StringIO(body) sigs = gpgContext.sign(plaintext, signature, gpgme.SIG_MODE_CLEAR) if (len(sigs) != 1): logging.warn('Failed to sign message!') signature.seek(0) [mailbody.write(l) for l in signature] else: mailbody.write(body) # Only log the email text (including all headers) if SAFE_LOGGING is # disabled: if not Util.safe_logging: f.seek(0) logging.debug("Email contents:\n%s" % f.read()) else: logging.debug("Email text for %r created." % Util.logSafely(clientAddr)) f.seek(0) return clientAddr, f
bridgeFilterRules=bridgeFilterRules) # Handle rate limited email except TooSoonEmail, e: logging.info("Got a mail too frequently; warning %r: %s.", Util.logSafely(clientAddr), e) # Compose a warning email # MAX_EMAIL_RATE is in seconds, convert to hours body = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600) return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID, gpgContext=ctx.gpgContext) except IgnoreEmail, e: logging.info("Got a mail too frequently; ignoring %r: %s.", Util.logSafely(clientAddr), e) return None, None except BadEmail, e: logging.info("Got a mail from a bad email address %r: %s.", Util.logSafely(clientAddr), e) return None, None if bridges: with_fp = ctx.cfg.EMAIL_INCLUDE_FINGERPRINTS answer = "".join(" %s\n" %b.getConfigLine( includeFingerprint=with_fp, addressClass=addressClass, transport=transport, request=clientAddr ) for b in bridges)
def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. emailaddress -- the user's email address, as given in a from line. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ if not bridgeFilterRules: bridgeFilterRules = [] now = time.time() try: emailaddress = normalizeEmail(emailaddress, self.domainmap, self.domainrules) except BadEmail: return [] #XXXX log the exception if emailaddress is None: return [] #XXXX raise an exception. db = bridgedb.Storage.getDB() wasWarned = db.getWarnedEmail(emailaddress) lastSaw = db.getEmailTime(emailaddress) if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now: if wasWarned: logging.info( "Got a request for bridges from %r; we already " "sent a warning. Ignoring.", Util.logSafely(emailaddress)) raise IgnoreEmail("Client was warned", Util.logSafely(emailaddress)) else: db.setWarnedEmail(emailaddress, True, now) db.commit() logging.info( "Got a request for bridges from %r; we already " "answered one within the last %d seconds. Warning.", Util.logSafely(emailaddress), MAX_EMAIL_RATE) raise TooSoonEmail("Too many emails; wait till later", emailaddress) # warning period is over elif wasWarned: db.setWarnedEmail(emailaddress, False) pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) ring = None ruleset = frozenset(bridgeFilterRules) if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _, ring = self.splitter.filterRings[ruleset] else: # cache miss, add new ring logging.debug("Cache miss %s" % ruleset) # add new ring key1 = bridgedb.Bridges.get_hmac(self.splitter.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # debug log: cache miss self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) result = ring.getBridges( pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N)) db.setEmailTime(emailaddress, now) db.commit() return result
def getBridgeRequestAnswer(self, request): """ returns a response to a bridge request """ interval = self.schedule.getInterval(time.time()) bridges = ( ) ip = None countryCode = None if self.useForwardedHeader: h = request.getHeader("X-Forwarded-For") if h: ip = h.split(",")[-1].strip() if not bridgedb.Bridges.is_valid_ip(ip): logging.warn("Got weird forwarded-for value %r",h) ip = None else: ip = request.getClientIP() if geoip: countryCode = geoip.country_code_by_addr(ip) # set locale setLocaleFromRequestHeader(request) format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg # do want any options? transport = ipv6 = unblocked = False ipv6 = request.args.get("ipv6", False) if ipv6: ipv6 = True # if anything after ?ipv6= try: # validate method name transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*', request.args.get("transport")[0]).group() except (TypeError, IndexError, AttributeError): transport = None try: unblocked = re.match('[a-zA-Z]{2,4}', request.args.get("unblocked")[0]).group() except (TypeError, IndexError, AttributeError): unblocked = False rules = [] if ip: if ipv6: rules.append(filterBridgesByIP6) addressClass = IPv6Address else: rules.append(filterBridgesByIP4) addressClass = IPv4Address if transport: #XXX: A cleaner solution would differentiate between # addresses by protocol rather than have separate lists # Tor to be a transport, and selecting between them. rules = [filterBridgesByTransport(transport, addressClass)] if unblocked: rules.append(filterBridgesByNotBlockedIn(unblocked, addressClass, transport)) bridges = self.distributor.getBridgesForIP(ip, interval, self.nBridgesToGive, countryCode, bridgeFilterRules=rules) answer = None if bridges: answer = "".join(" %s\n" % b.getConfigLine( includeFingerprint=self.includeFingerprints, addressClass=addressClass, transport=transport, request=bridgedb.Dist.uniformMap(ip) ) for b in bridges) logging.info("Replying to web request from %s. Parameters were %r", Util.logSafely(ip), request.args) if format == 'plain': request.setHeader("Content-Type", "text/plain") return answer else: request.setHeader("Content-Type", "text/html; charset=utf-8") return lookup.get_template('bridges.html').render(answer=answer)
def getMailResponse(lines, ctx): """Given a list of lines from an incoming email message, and a MailContext object, parse the email and decide what to do in response. If we want to answer, return a 2-tuple containing the address that will receive the response, and a readable filelike object containing the response. Return None,None if we shouldn't answer. """ # Extract data from the headers. msg = rfc822.Message(MailFile(lines)) subject = msg.getheader("Subject", None) if not subject: subject = "[no subject]" clientFromAddr = msg.getaddr("From") clientSenderAddr = msg.getaddr("Sender") # RFC822 requires at least one 'To' address clientToList = msg.getaddrlist("To") clientToaddr = getBridgeDBEmailAddrFromList(ctx, clientToList) msgID = msg.getheader("Message-ID", None) if clientSenderAddr and clientSenderAddr[1]: clientAddr = clientSenderAddr[1] elif clientFromAddr and clientFromAddr[1]: clientAddr = clientFromAddr[1] else: logging.info("No From or Sender header on incoming mail.") return None,None # Look up the locale part in the 'To:' address, if there is one and get # the appropriate Translation object lang = getLocaleFromPlusAddr(clientToaddr) t = I18n.getLang(lang) try: _, addrdomain = bridgedb.Dist.extractAddrSpec(clientAddr.lower()) except BadEmail: logging.info("Ignoring bad address on incoming email.") return None,None if not addrdomain: logging.info("Couldn't parse domain from %r", Util.logSafely(clientAddr)) if addrdomain and ctx.cfg.EMAIL_DOMAIN_MAP: addrdomain = ctx.cfg.EMAIL_DOMAIN_MAP.get(addrdomain, addrdomain) if addrdomain not in ctx.cfg.EMAIL_DOMAINS: logging.info("Unrecognized email domain %r", Util.logSafely(addrdomain)) return None,None rules = ctx.cfg.EMAIL_DOMAIN_RULES.get(addrdomain, []) if 'dkim' in rules: # getheader() returns the last of a given kind of header; we want # to get the first, so we use getheaders() instead. dkimHeaders = msg.getheaders("X-DKIM-Authentication-Results") dkimHeader = "<no header>" if dkimHeaders: dkimHeader = dkimHeaders[0] if not dkimHeader.startswith("pass"): logging.info("Got a bad dkim header (%r) on an incoming mail; " "rejecting it.", dkimHeader) return None, None # Was the magic string included #for ln in lines: # if ln.strip().lower() in ("get bridges", "subject: get bridges"): # break #else: # logging.info("Got a mail from %r with no bridge request; dropping", # clientAddr) # return None,None # Figure out which bridges to send unblocked = transport = ipv6 = skippedheaders = False bridgeFilterRules = [] addressClass = None for ln in lines: # ignore all lines before the subject header if "subject" in ln.strip().lower(): skippedheaders = True if not skippedheaders: continue if "ipv6" in ln.strip().lower(): ipv6 = True if "transport" in ln.strip().lower(): try: transport = re.search("transport ([_a-zA-Z][_a-zA-Z0-9]*)", ln).group(1).strip() except (TypeError, AttributeError): transport = None logging.debug("Got request for transport: %s" % transport) if "unblocked" in ln.strip().lower(): try: unblocked = re.search("unblocked ([a-zA-Z]{2,4})", ln).group(1).strip() except (TypeError, AttributeError): transport = None if ipv6: bridgeFilterRules.append(filterBridgesByIP6) addressClass = IPv6Address else: bridgeFilterRules.append(filterBridgesByIP4) addressClass = IPv4Address if transport: bridgeFilterRules = [filterBridgesByTransport(transport, addressClass)] if unblocked: rules.append(filterBridgesByNotBlockedIn(unblocked, addressClass, transport)) try: interval = ctx.schedule.getInterval(time.time()) bridges = ctx.distributor.getBridgesForEmail(clientAddr, interval, ctx.N, countryCode=None, bridgeFilterRules=bridgeFilterRules) # Handle rate limited email except TooSoonEmail, e: logging.info("Got a mail too frequently; warning %r: %s.", Util.logSafely(clientAddr), e) # Compose a warning email # MAX_EMAIL_RATE is in seconds, convert to hours body = buildSpamWarningTemplate(t) % (bridgedb.Dist.MAX_EMAIL_RATE / 3600) return composeEmail(ctx.fromAddr, clientAddr, subject, body, msgID, gpgContext=ctx.gpgContext)
def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. ip -- the user's IP address, as a dotted quad. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ if not bridgeFilterRules: bridgeFilterRules = [] logging.debug("getBridgesForIP(%s, %s, %s, %s", Util.logSafely(ip), epoch, N, bridgeFilterRules) if not len(self.splitter): logging.debug("bailing without splitter") return [] area = self.areaMapper(ip) logging.info("area is %s", Util.logSafely(area)) key1 = '' pos = 0 n = self.nClusters # only one of ip categories or area clustering is active # try to match the request to an ip category for category in self.categories: # IP Categories if category.contains(ip): g = filterAssignBridgesToRing( self.splitter.hmac, self.nClusters + len(self.categories), n) bridgeFilterRules.append(g) logging.info("category<%s>%s", epoch, Util.logSafely(area)) pos = self.areaOrderHmac("category<%s>%s" % (epoch, area)) key1 = bridgedb.Bridges.get_hmac( self.splitter.key, "Order-Bridges-In-Ring-%d" % n) break n += 1 # if no category matches, use area clustering else: # IP clustering h = int(self.areaClusterHmac(area)[:8], 16) # length of numClusters clusterNum = h % self.nClusters g = filterAssignBridgesToRing( self.splitter.hmac, self.nClusters + len(self.categories), clusterNum) bridgeFilterRules.append(g) pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) key1 = bridgedb.Bridges.get_hmac( self.splitter.key, "Order-Bridges-In-Ring-%d" % clusterNum) logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) # try to find a cached copy ruleset = frozenset(bridgeFilterRules) # See if we have a cached copy of the ring, # otherwise, add a new ring and populate it if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _, ring = self.splitter.filterRings[ruleset] # else create the ring and populate it else: logging.debug("Cache miss %s" % ruleset) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) # get an appropriate number of bridges return ring.getBridges( pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))
def getBridgesForEmail(self, emailaddress, epoch, N=1, parameters=None, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. emailaddress -- the user's email address, as given in a from line. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ if not bridgeFilterRules: bridgeFilterRules=[] now = time.time() try: emailaddress = normalizeEmail(emailaddress, self.domainmap, self.domainrules) except BadEmail: return [] #XXXX log the exception if emailaddress is None: return [] #XXXX raise an exception. db = bridgedb.Storage.getDB() wasWarned = db.getWarnedEmail(emailaddress) lastSaw = db.getEmailTime(emailaddress) if lastSaw is not None and lastSaw + MAX_EMAIL_RATE >= now: if wasWarned: logging.info("Got a request for bridges from %r; we already " "sent a warning. Ignoring.", Util.logSafely(emailaddress)) raise IgnoreEmail("Client was warned", Util.logSafely(emailaddress)) else: db.setWarnedEmail(emailaddress, True, now) db.commit() logging.info("Got a request for bridges from %r; we already " "answered one within the last %d seconds. Warning.", Util.logSafely(emailaddress), MAX_EMAIL_RATE) raise TooSoonEmail("Too many emails; wait till later", emailaddress) # warning period is over elif wasWarned: db.setWarnedEmail(emailaddress, False) pos = self.emailHmac("<%s>%s" % (epoch, emailaddress)) ring = None ruleset = frozenset(bridgeFilterRules) if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _,ring = self.splitter.filterRings[ruleset] else: # cache miss, add new ring logging.debug("Cache miss %s" % ruleset) # add new ring key1 = bridgedb.Bridges.get_hmac(self.splitter.key, "Order-Bridges-In-Ring") ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) # debug log: cache miss self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) result = ring.getBridges(pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N)) db.setEmailTime(emailaddress, now) db.commit() return result
def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. ip -- the user's IP address, as a dotted quad. epoch -- the time period when we got this request. This can be any string, so long as it changes with every period. N -- the number of bridges to try to give back. """ if not bridgeFilterRules: bridgeFilterRules=[] logging.debug("getBridgesForIP(%s, %s, %s, %s", Util.logSafely(ip), epoch, N, bridgeFilterRules) if not len(self.splitter): logging.debug("bailing without splitter") return [] area = self.areaMapper(ip) logging.info("area is %s", Util.logSafely(area)) key1 = '' pos = 0 n = self.nClusters # only one of ip categories or area clustering is active # try to match the request to an ip category for category in self.categories: # IP Categories if category.contains(ip): g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), n) bridgeFilterRules.append(g) logging.info("category<%s>%s", epoch, Util.logSafely(area)) pos = self.areaOrderHmac("category<%s>%s" % (epoch, area)) key1 = bridgedb.Bridges.get_hmac(self.splitter.key, "Order-Bridges-In-Ring-%d"%n) break; n += 1 # if no category matches, use area clustering else: # IP clustering h = int( self.areaClusterHmac(area)[:8], 16) # length of numClusters clusterNum = h % self.nClusters g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), clusterNum) bridgeFilterRules.append(g) pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) key1 = bridgedb.Bridges.get_hmac(self.splitter.key, "Order-Bridges-In-Ring-%d"%clusterNum) logging.debug("bridgeFilterRules: %s" % bridgeFilterRules) # try to find a cached copy ruleset = frozenset(bridgeFilterRules) # See if we have a cached copy of the ring, # otherwise, add a new ring and populate it if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _,ring = self.splitter.filterRings[ruleset] # else create the ring and populate it else: logging.debug("Cache miss %s" % ruleset) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) # get an appropriate number of bridges return ring.getBridges(pos, getNumBridgesPerAnswer(ring, max_bridges_per_answer=N))
def getBridgeRequestAnswer(self, request): """Respond to a client HTTP request for bridges. :type request: :api:`twisted.web.http.Request` :param request: A ``Request`` object containing the HTTP method, full URI, and any URL/POST arguments and headers present. :rtype: str :returns: A plaintext or HTML response to serve. """ # XXX why are we getting the interval if our distributor might be # using bridgedb.Time.NoSchedule? interval = self.schedule.getInterval(time.time()) bridges = ( ) ip = None countryCode = None # XXX this code is duplicated in CaptchaProtectedResource if self.useForwardedHeader: h = request.getHeader("X-Forwarded-For") if h: ip = h.split(",")[-1].strip() if not bridgedb.Bridges.is_valid_ip(ip): logging.warn("Got weird forwarded-for value %r",h) ip = None else: ip = request.getClientIP() # XXX This can also be a separate function # XXX if the ip is None, this throws an exception if geoip: countryCode = geoip.country_code_by_addr(ip) if countryCode: logging.debug("Client request from GeoIP CC: %s" % countryCode) rtl = usingRTLLang(request) if rtl: logging.debug("Rendering RTL response.") # XXX separate function again format = request.args.get("format", None) if format and len(format): format = format[0] # choose the first arg # do want any options? transport = ipv6 = unblocked = False ipv6 = request.args.get("ipv6", False) if ipv6: ipv6 = True # if anything after ?ipv6= # XXX oh dear hell. why not check for the '?transport=' arg before # regex'ing? And why not compile the regex once, somewhere outside # this function and class? try: # validate method name transport = re.match('[_a-zA-Z][_a-zA-Z0-9]*', request.args.get("transport")[0]).group() except (TypeError, IndexError, AttributeError): transport = None try: unblocked = re.match('[a-zA-Z]{2,4}', request.args.get("unblocked")[0]).group() except (TypeError, IndexError, AttributeError): unblocked = False logging.info("Replying to web request from %s. Parameters were %r" % (Util.logSafely(ip), request.args)) rules = [] bridgeLines = None if ip: if ipv6: rules.append(filterBridgesByIP6) addressClass = IPv6Address else: rules.append(filterBridgesByIP4) addressClass = IPv4Address if transport: #XXX: A cleaner solution would differentiate between # addresses by protocol rather than have separate lists # Tor to be a transport, and selecting between them. rules = [filterBridgesByTransport(transport, addressClass)] if unblocked: rules.append(filterBridgesByNotBlockedIn(unblocked, addressClass, transport)) bridges = self.distributor.getBridgesForIP(ip, interval, self.nBridgesToGive, countryCode, bridgeFilterRules=rules) bridgeLines = "".join(" %s\n" % b.getConfigLine( includeFingerprint=self.includeFingerprints, addressClass=addressClass, transport=transport, request=bridgedb.Dist.uniformMap(ip) ) for b in bridges) answer = self.renderAnswer(request, bridgeLines, rtl, format) return answer
def getBridgesForIP(self, ip, epoch, N=1, countryCode=None, bridgeFilterRules=None): """Return a list of bridges to give to a user. :param str ip: The user's IP address, as a dotted quad. :param str epoch: The time period when we got this request. This can be any string, so long as it changes with every period. :param int N: The number of bridges to try to give back. (default: 1) :param str countryCode: DOCDOC (default: None) :param list bridgeFilterRules: A list of callables used filter the bridges returned in the response to the client. See :mod:`~bridgedb.Filters`. :rtype: list :return: A list of :class:`~bridgedb.Bridges.Bridge`s to include in the response. See :meth:`bridgedb.HTTPServer.WebResource.getBridgeRequestAnswer` for an example of how this is used. """ logging.info("Attempting to return %d bridges to client %s..." % (N, Util.logSafely(ip))) if not bridgeFilterRules: bridgeFilterRules=[] if not len(self.splitter): logging.warn("Bailing! Splitter has zero bridges!") return [] logging.debug("Bridges in splitter:\t%d" % len(self.splitter)) logging.debug("Client request epoch:\t%s" % epoch) logging.debug("Active bridge filters:\t%s" % ' '.join([x.func_name for x in bridgeFilterRules])) area = self.areaMapper(ip) logging.debug("IP mapped to area:\t%s" % Util.logSafely("{0}.0/24".format(area))) key1 = '' pos = 0 n = self.nClusters # only one of ip categories or area clustering is active # try to match the request to an ip category for category in self.categories: # IP Categories if category.contains(ip): g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), n) bridgeFilterRules.append(g) logging.info("category<%s>%s", epoch, Util.logSafely(area)) pos = self.areaOrderHmac("category<%s>%s" % (epoch, area)) key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % n) break n += 1 # if no category matches, use area clustering else: # IP clustering h = int( self.areaClusterHmac(area)[:8], 16) # length of numClusters clusterNum = h % self.nClusters g = filterAssignBridgesToRing(self.splitter.hmac, self.nClusters + len(self.categories), clusterNum) bridgeFilterRules.append(g) pos = self.areaOrderHmac("<%s>%s" % (epoch, area)) key1 = getHMAC(self.splitter.key, "Order-Bridges-In-Ring-%d" % clusterNum) # try to find a cached copy ruleset = frozenset(bridgeFilterRules) # See if we have a cached copy of the ring, # otherwise, add a new ring and populate it if ruleset in self.splitter.filterRings.keys(): logging.debug("Cache hit %s" % ruleset) _,ring = self.splitter.filterRings[ruleset] # else create the ring and populate it else: logging.debug("Cache miss %s" % ruleset) ring = bridgedb.Bridges.BridgeRing(key1, self.answerParameters) self.splitter.addRing(ring, ruleset, filterBridgesByRules(bridgeFilterRules), populate_from=self.splitter.bridges) # get an appropriate number of bridges numBridgesToReturn = getNumBridgesPerAnswer(ring, max_bridges_per_answer=N) answer = ring.getBridges(pos, numBridgesToReturn) return answer