Exemplo n.º 1
0
 def remove_handler(self, fd):
     self._handlers.pop(fd, None)
     self._events.pop(fd, None)
     try:
         self._impl.unregister(fd)
     except Exception:
         gen_log.debug("Error deleting fd from IOLoop", exc_info=True)
Exemplo n.º 2
0
 def accept_connection(self):
     try:
         self._handle_websocket_headers()
         self._accept_connection()
     except ValueError:
         gen_log.debug("Malformed WebSocket request received", exc_info=True)
         self._abort()
         return
Exemplo n.º 3
0
 def _handle_challenge(self, challenge):
     try:
         challenge_response = self.challenge_response(challenge)
     except ValueError:
         gen_log.debug("Malformed key data in WebSocket request")
         self._abort()
         return
     self._write_response(challenge_response)
Exemplo n.º 4
0
 def close(self, all_fds=False):
     with self._callback_lock:
         self._closing = True
     self.remove_handler(self._waker.fileno())
     if all_fds:
         for fd in self._handlers.keys():
             try:
                 close_method = getattr(fd, 'close', None)
                 if close_method is not None:
                     close_method()
                 else:
                     os.close(fd)
             except Exception:
                 gen_log.debug("error closing fd %s", fd, exc_info=True)
     self._waker.close()
     self._impl.close()
Exemplo n.º 5
0
 def fetch_impl(self, request, callback):
     key = object()
     self.queue.append((key, request, callback))
     if not len(self.active) < self.max_clients:
         timeout_handle = self.io_loop.add_timeout(
             self.io_loop.time() + min(request.connect_timeout,
                                       request.request_timeout),
             functools.partial(self._on_timeout, key))
     else:
         timeout_handle = None
     self.waiting[key] = (request, callback, timeout_handle)
     self._process_queue()
     if self.queue:
         gen_log.debug("max_clients limit reached, request queued. "
                       "%d active, %d queued requests." % (
                           len(self.active), len(self.queue)))
Exemplo n.º 6
0
def load_gettext_translations(directory, domain):
    """Loads translations from `gettext`'s locale tree

    Locale tree is similar to system's ``/usr/share/locale``, like::

        {directory}/{lang}/LC_MESSAGES/{domain}.mo

    Three steps are required to have you app translated:

    1. Generate POT translation file::

        xgettext --language=Python --keyword=_:1,2 -d mydomain file1.py file2.html etc

    2. Merge against existing POT file::

        msgmerge old.po mydomain.po > new.po

    3. Compile::

        msgfmt mydomain.po -o {directory}/pt_BR/LC_MESSAGES/mydomain.mo
    """
    import gettext
    global _translations
    global _supported_locales
    global _use_gettext
    _translations = {}
    for lang in os.listdir(directory):
        if lang.startswith('.'):
            continue  # skip .svn, etc
        if os.path.isfile(os.path.join(directory, lang)):
            continue
        try:
            os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain + ".mo"))
            _translations[lang] = gettext.translation(domain, directory,
                                                      languages=[lang])
        except Exception as e:
            gen_log.error("Cannot load translation for '%s': %s", lang, str(e))
            continue
    _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])
    _use_gettext = True
    gen_log.debug("Supported locales: %s", sorted(_supported_locales))
Exemplo n.º 7
0
    def accept_connection(self):
        try:
            self._handle_websocket_headers()
        except ValueError:
            gen_log.debug("Malformed WebSocket request received")
            self._abort()
            return

        scheme = self.handler.get_websocket_scheme()

        # draft76 only allows a single subprotocol
        subprotocol_header = ''
        subprotocol = self.request.headers.get("Sec-WebSocket-Protocol", None)
        if subprotocol:
            selected = self.handler.select_subprotocol([subprotocol])
            if selected:
                assert selected == subprotocol
                subprotocol_header = "Sec-WebSocket-Protocol: %s\r\n" % selected

        # Write the initial headers before attempting to read the challenge.
        # This is necessary when using proxies (such as HAProxy), which
        # need to see the Upgrade headers before passing through the
        # non-HTTP traffic that follows.
        self.stream.write(webalchemy.tornado.escape.utf8(
            "HTTP/1.1 101 WebSocket Protocol Handshake\r\n"
            "Upgrade: WebSocket\r\n"
            "Connection: Upgrade\r\n"
            "Server: TornadoServer/%(version)s\r\n"
            "Sec-WebSocket-Origin: %(origin)s\r\n"
            "Sec-WebSocket-Location: %(scheme)s://%(host)s%(uri)s\r\n"
            "%(subprotocol)s"
            "\r\n" % (dict(
                version=webalchemy.tornado.version,
                origin=self.request.headers["Origin"],
                scheme=scheme,
                host=self.request.host,
                uri=self.request.uri,
                subprotocol=subprotocol_header))))
        self.stream.read_bytes(8, self._handle_challenge)
Exemplo n.º 8
0
def _curl_debug(debug_type, debug_msg):
    debug_types = ('I', '<', '>', '<', '>')
    if debug_type == 0:
        gen_log.debug('%s', debug_msg.strip())
    elif debug_type in (1, 2):
        for line in debug_msg.splitlines():
            gen_log.debug('%s %s', debug_types[debug_type], line)
    elif debug_type == 4:
        gen_log.debug('%s %r', debug_types[debug_type], debug_msg)
Exemplo n.º 9
0
def _curl_setup_request(curl, request, buffer, headers):
    curl.setopt(pycurl.URL, native_str(request.url))

    # libcurl's magic "Expect: 100-continue" behavior causes delays
    # with servers that don't support it (which include, among others,
    # Google's OpenID endpoint).  Additionally, this behavior has
    # a bug in conjunction with the curl_multi_socket_action API
    # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976),
    # which increases the delays.  It's more trouble than it's worth,
    # so just turn off the feature (yes, setting Expect: to an empty
    # value is the official way to disable this)
    if "Expect" not in request.headers:
        request.headers["Expect"] = ""

    # libcurl adds Pragma: no-cache by default; disable that too
    if "Pragma" not in request.headers:
        request.headers["Pragma"] = ""

    # Request headers may be either a regular dict or HTTPHeaders object
    if isinstance(request.headers, httputil.HTTPHeaders):
        curl.setopt(pycurl.HTTPHEADER,
                    [native_str("%s: %s" % i) for i in request.headers.get_all()])
    else:
        curl.setopt(pycurl.HTTPHEADER,
                    [native_str("%s: %s" % i) for i in request.headers.items()])

    if request.header_callback:
        curl.setopt(pycurl.HEADERFUNCTION,
                    lambda line: request.header_callback(native_str(line)))
    else:
        curl.setopt(pycurl.HEADERFUNCTION,
                    lambda line: _curl_header_callback(headers,
                                                       native_str(line)))
    if request.streaming_callback:
        write_function = request.streaming_callback
    else:
        write_function = buffer.write
    if bytes_type is str:  # py2
        curl.setopt(pycurl.WRITEFUNCTION, write_function)
    else:  # py3
        # Upstream pycurl doesn't support py3, but ubuntu 12.10 includes
        # a fork/port.  That version has a bug in which it passes unicode
        # strings instead of bytes to the WRITEFUNCTION.  This means that
        # if you use a WRITEFUNCTION (which webalchemy.tornado always does), you cannot
        # download arbitrary binary data.  This needs to be fixed in the
        # ported pycurl package, but in the meantime this lambda will
        # make it work for downloading (utf8) text.
        curl.setopt(pycurl.WRITEFUNCTION, lambda s: write_function(utf8(s)))
    curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects)
    curl.setopt(pycurl.MAXREDIRS, request.max_redirects)
    curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout))
    curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout))
    if request.user_agent:
        curl.setopt(pycurl.USERAGENT, native_str(request.user_agent))
    else:
        curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)")
    if request.network_interface:
        curl.setopt(pycurl.INTERFACE, request.network_interface)
    if request.use_gzip:
        curl.setopt(pycurl.ENCODING, "gzip,deflate")
    else:
        curl.setopt(pycurl.ENCODING, "none")
    if request.proxy_host and request.proxy_port:
        curl.setopt(pycurl.PROXY, request.proxy_host)
        curl.setopt(pycurl.PROXYPORT, request.proxy_port)
        if request.proxy_username:
            credentials = '%s:%s' % (request.proxy_username,
                                     request.proxy_password)
            curl.setopt(pycurl.PROXYUSERPWD, credentials)
    else:
        curl.setopt(pycurl.PROXY, '')
        curl.unsetopt(pycurl.PROXYUSERPWD)
    if request.validate_cert:
        curl.setopt(pycurl.SSL_VERIFYPEER, 1)
        curl.setopt(pycurl.SSL_VERIFYHOST, 2)
    else:
        curl.setopt(pycurl.SSL_VERIFYPEER, 0)
        curl.setopt(pycurl.SSL_VERIFYHOST, 0)
    if request.ca_certs is not None:
        curl.setopt(pycurl.CAINFO, request.ca_certs)
    else:
        # There is no way to restore pycurl.CAINFO to its default value
        # (Using unsetopt makes it reject all certificates).
        # I don't see any way to read the default value from python so it
        # can be restored later.  We'll have to just leave CAINFO untouched
        # if no ca_certs file was specified, and require that if any
        # request uses a custom ca_certs file, they all must.
        pass

    if request.allow_ipv6 is False:
        # Curl behaves reasonably when DNS resolution gives an ipv6 address
        # that we can't reach, so allow ipv6 unless the user asks to disable.
        # (but see version check in _process_queue above)
        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4)
    else:
        curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_WHATEVER)

    # Set the request method through curl's irritating interface which makes
    # up names for almost every single method
    curl_options = {
        "GET": pycurl.HTTPGET,
        "POST": pycurl.POST,
        "PUT": pycurl.UPLOAD,
        "HEAD": pycurl.NOBODY,
    }
    custom_methods = set(["DELETE", "OPTIONS", "PATCH"])
    for o in curl_options.values():
        curl.setopt(o, False)
    if request.method in curl_options:
        curl.unsetopt(pycurl.CUSTOMREQUEST)
        curl.setopt(curl_options[request.method], True)
    elif request.allow_nonstandard_methods or request.method in custom_methods:
        curl.setopt(pycurl.CUSTOMREQUEST, request.method)
    else:
        raise KeyError('unknown method ' + request.method)

    # Handle curl's cryptic options for every individual HTTP method
    if request.method in ("POST", "PUT"):
        if request.body is None:
            raise AssertionError(
                'Body must not be empty for "%s" request'
                % request.method)

        request_buffer = BytesIO(utf8(request.body))
        curl.setopt(pycurl.READFUNCTION, request_buffer.read)
        if request.method == "POST":
            def ioctl(cmd):
                if cmd == curl.IOCMD_RESTARTREAD:
                    request_buffer.seek(0)
            curl.setopt(pycurl.IOCTLFUNCTION, ioctl)
            curl.setopt(pycurl.POSTFIELDSIZE, len(request.body))
        else:
            curl.setopt(pycurl.INFILESIZE, len(request.body))
    elif request.method == "GET":
        if request.body is not None:
            raise AssertionError('Body must be empty for GET request')

    if request.auth_username is not None:
        userpwd = "%s:%s" % (request.auth_username, request.auth_password or '')

        if request.auth_mode is None or request.auth_mode == "basic":
            curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC)
        elif request.auth_mode == "digest":
            curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_DIGEST)
        else:
            raise ValueError("Unsupported auth_mode %s" % request.auth_mode)

        curl.setopt(pycurl.USERPWD, native_str(userpwd))
        gen_log.debug("%s %s (username: %r)", request.method, request.url,
                      request.auth_username)
    else:
        curl.unsetopt(pycurl.USERPWD)
        gen_log.debug("%s %s", request.method, request.url)

    if request.client_cert is not None:
        curl.setopt(pycurl.SSLCERT, request.client_cert)

    if request.client_key is not None:
        curl.setopt(pycurl.SSLKEY, request.client_key)

    if threading.activeCount() > 1:
        # libcurl/pycurl is not thread-safe by default.  When multiple threads
        # are used, signals should be disabled.  This has the side effect
        # of disabling DNS timeouts in some environments (when libcurl is
        # not linked against ares), so we don't do it when there is only one
        # thread.  Applications that use many short-lived threads may need
        # to set NOSIGNAL manually in a prepare_curl_callback since
        # there may not be any other threads running at the time we call
        # threading.activeCount.
        curl.setopt(pycurl.NOSIGNAL, 1)
    if request.prepare_curl_callback is not None:
        request.prepare_curl_callback(curl)
Exemplo n.º 10
0
def load_translations(directory):
    """Loads translations from CSV files in a directory.

    Translations are strings with optional Python-style named placeholders
    (e.g., ``My name is %(name)s``) and their associated translations.

    The directory should have translation files of the form ``LOCALE.csv``,
    e.g. ``es_GT.csv``. The CSV files should have two or three columns: string,
    translation, and an optional plural indicator. Plural indicators should
    be one of "plural" or "singular". A given string can have both singular
    and plural forms. For example ``%(name)s liked this`` may have a
    different verb conjugation depending on whether %(name)s is one
    name or a list of names. There should be two rows in the CSV file for
    that string, one with plural indicator "singular", and one "plural".
    For strings with no verbs that would change on translation, simply
    use "unknown" or the empty string (or don't include the column at all).

    The file is read using the `csv` module in the default "excel" dialect.
    In this format there should not be spaces after the commas.

    Example translation ``es_LA.csv``::

        "I love you","Te amo"
        "%(name)s liked this","A %(name)s les gustó esto","plural"
        "%(name)s liked this","A %(name)s le gustó esto","singular"

    """
    global _translations
    global _supported_locales
    _translations = {}
    for path in os.listdir(directory):
        if not path.endswith(".csv"):
            continue
        locale, extension = path.split(".")
        if not re.match("[a-z]+(_[A-Z]+)?$", locale):
            gen_log.error("Unrecognized locale %r (path: %s)", locale,
                          os.path.join(directory, path))
            continue
        full_path = os.path.join(directory, path)
        try:
            # python 3: csv.reader requires a file open in text mode.
            # Force utf8 to avoid dependence on $LANG environment variable.
            f = open(full_path, "r", encoding="utf-8")
        except TypeError:
            # python 2: files return byte strings, which are decoded below.
            f = open(full_path, "r")
        _translations[locale] = {}
        for i, row in enumerate(csv.reader(f)):
            if not row or len(row) < 2:
                continue
            row = [escape.to_unicode(c).strip() for c in row]
            english, translation = row[:2]
            if len(row) > 2:
                plural = row[2] or "unknown"
            else:
                plural = "unknown"
            if plural not in ("plural", "singular", "unknown"):
                gen_log.error("Unrecognized plural indicator %r in %s line %d",
                              plural, path, i + 1)
                continue
            _translations[locale].setdefault(plural, {})[english] = translation
        f.close()
    _supported_locales = frozenset(list(_translations.keys()) + [_default_locale])
    gen_log.debug("Supported locales: %s", sorted(_supported_locales))