Exemplo n.º 1
0
    def handle_application_error(self, environ, start_response):
        status = "500 Internal Server Error"
        headers = Headers([])

        # Package the exception info as into a special header and
        # send it to the client
        type, exc, tb = sys.exc_info()

        tbfile = StringIO()

        traceback.print_exc(file=tbfile)
        headers['Content-Type'] = 'text/plain; charset=utf-8'

        LOG.debug("Packing traceback context into debug header: %s",
                  self.debug_header)
        debug_header = self.pack_header(Traceback(tb))
        LOG.debug("Debug header (%d bytes): %s",
                  len(debug_header), debug_header)
        headers[self.debug_header] = debug_header

        app_uri = application_uri(environ)
        headers["Location"] = app_uri[:-1] + self.debug_uri

        start_response(status, headers.items())
        return [tbfile.getvalue().encode('utf-8')]
Exemplo n.º 2
0
        def patched_start_response(status, headers, exc_info=None):
            # if self._should_handle(headers)
            wsgi_headers = Headers(headers)

            # If we're debugging, or the response already has an expires
            # header, just skip this.
            if not self.debug and "Expires" not in wsgi_headers:
                mime = wsgi_headers.get("Content-Type", "*").split(";")[0]

                # If the mime type is explicitly called out, use the expire
                # delay specified.
                if mime in self.expire_seconds:
                    expire_time = self.make_expire_time_for(mime)

                # If there's a catch-all wildcard delay, use that.
                elif "*" in self.expire_seconds:
                    expire_time = self.make_expire_time_for("*")

                # Otherwise, don't set the header.
                else:
                    expire_time = None

                if expire_time is not None:
                    log.debug("Adding expires header value: " + expire_time)
                    headers.append(("Expires", expire_time))

            return start_response(status, headers, exc_info)
Exemplo n.º 3
0
def thread_app(env, resp):
    path = env['PATH_INFO']
    # utils.log('thread_app', path)
    m = thread_re.match(path)
    board, datkey = m.group(1), m.group(2)

    key = keylib.get_filekey(datkey)
    data = cache.Cache(key)
    data.load()

    if check_get_cache(env):
        if not data.exists() or len(data) == 0:
            # when first access, load data from network
            data.search()

        elif _count_is_update(key):
            # update thread
            # limit `data.search` calling. it's slow!
            threading.Thread(target=data.search, daemon=True).start()

    if not data.exists():
        resp('404 Not Found', [('Content-Type', 'text/plain; charset=Shift_JIS')])
        return [b'404 Not Found']

    thread = dat.make_dat(data, env, board)

    headers = Headers([('Content-Type', 'text/plain; charset=Shift_JIS')])
    last_m = eutils.formatdate(data.stamp)
    headers['Last-Modified'] = last_m
    resp("200 OK", headers.items())

    return (c.encode('cp932', 'replace') for c in thread)
Exemplo n.º 4
0
        def patched_start_response(status, headers, exc_info=None):
            # if self._should_handle(headers)
            wsgi_headers = Headers(headers)

            # If we're debugging, or the response already has an expires
            # header, just skip this.
            log.debug('Skipping expired headers' if self.debug else 'Calculating expires headers')
            if not self.debug and 'Expires' not in wsgi_headers:
                mime = wsgi_headers.get('Content-Type', '*').split(';')[0]
                log.debug('See mime type ' + mime)

                # If the mime type is explicitly called out, use the expire
                # delay specified.
                if mime in self.expire_seconds:
                    log.debug('Matched mimetype exactly.')
                    expire_time = self.make_expire_time_for(mime)

                # If there's a catch-all wildcard delay, use that.
                elif '*' in self.expire_seconds:
                    log.debug('Matched mimetype with universal.')
                    expire_time = self.make_expire_time_for('*')

                # Otherwise, don't set the header.
                else:
                    log.debug('No mimetype match.')
                    expire_time = None

                if expire_time is not None:
                    log.debug('Adding expires header value: ' + expire_time)
                    headers.append(('Expires', expire_time))

            log.debug('-'*60)
            return start_response(status, headers, exc_info)
Exemplo n.º 5
0
def board_app(env, resp):
    path = env['PATH_INFO']
    m = board_re.match(path)
    board = m.group(1)
    message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja'))

    headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')])
    resp("200 OK", headers.items())

    board = utils.sanitize(utils.get_board(path))

    if board:
        fmt = '{logo} - {board} - {desc}'
    else:
        fmt = '{logo} - {desc}'

    text = fmt.format(logo=message['logo'], desc=message['description'], board=board)

    html = '''
        <!DOCTYPE html>
        <html><head>
        <meta http-equiv="content-type" content="text/html; charset=Shift_JIS">
        <title>{text}</title>
        <meta name="description" content="{text}">
        </head><body>
        <h1>{text}</h1>
        </body></html>
    '''.format(text=text)
    return [html.encode('cp932', 'replace')]
Exemplo n.º 6
0
    def testMappingInterface(self):
        test = [("x", "y")]
        self.assertEqual(len(Headers([])), 0)
        self.assertEqual(len(Headers(test[:])), 1)
        self.assertEqual(Headers(test[:]).keys(), ["x"])
        self.assertEqual(Headers(test[:]).values(), ["y"])
        self.assertEqual(Headers(test[:]).items(), test)
        self.assertIsNot(Headers(test).items(), test)  # must be copy!

        h = Headers([])
        del h["foo"]  # should not raise an error

        h["Foo"] = "bar"
        for m in h.has_key, h.__contains__, h.get, h.get_all, h.__getitem__:
            self.assertTrue(m("foo"))
            self.assertTrue(m("Foo"))
            self.assertTrue(m("FOO"))
            self.assertFalse(m("bar"))

        self.assertEqual(h["foo"], "bar")
        h["foo"] = "baz"
        self.assertEqual(h["FOO"], "baz")
        self.assertEqual(h.get_all("foo"), ["baz"])

        self.assertEqual(h.get("foo", "whee"), "baz")
        self.assertEqual(h.get("zoo", "whee"), "whee")
        self.assertEqual(h.setdefault("foo", "whee"), "baz")
        self.assertEqual(h.setdefault("zoo", "whee"), "whee")
        self.assertEqual(h["foo"], "baz")
        self.assertEqual(h["zoo"], "whee")
Exemplo n.º 7
0
    def testMappingInterface(self):
        test = [('x','y')]
        self.assertEqual(len(Headers([])),0)
        self.assertEqual(len(Headers(test[:])),1)
        self.assertEqual(Headers(test[:]).keys(), ['x'])
        self.assertEqual(Headers(test[:]).values(), ['y'])
        self.assertEqual(Headers(test[:]).items(), test)
        self.assertFalse(Headers(test).items() is test)  # must be copy!

        h=Headers([])
        del h['foo']   # should not raise an error

        h['Foo'] = 'bar'
        for m in h.__contains__, h.get, h.get_all, h.__getitem__:
            self.assertTrue(m('foo'))
            self.assertTrue(m('Foo'))
            self.assertTrue(m('FOO'))
            self.assertFalse(m('bar'))

        self.assertEqual(h['foo'],'bar')
        h['foo'] = 'baz'
        self.assertEqual(h['FOO'],'baz')
        self.assertEqual(h.get_all('foo'),['baz'])

        self.assertEqual(h.get("foo","whee"), "baz")
        self.assertEqual(h.get("zoo","whee"), "whee")
        self.assertEqual(h.setdefault("foo","whee"), "baz")
        self.assertEqual(h.setdefault("zoo","whee"), "whee")
        self.assertEqual(h["foo"],"baz")
        self.assertEqual(h["zoo"],"whee")
Exemplo n.º 8
0
def parse_http_headers(environ):
    h = Headers([])
    for k, v in environ.items():
        if k.startswith('HTTP_'):
            name = k[5:]
            h.add_header(name, v)
    return h
Exemplo n.º 9
0
    def getTileResponse(self, coord, extension, ignore_cached=False):
        """ Get status code, headers, and a tile binary for a given request layer tile.
        
            Arguments:
            - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
            - extension: filename extension to choose response type, e.g. "png" or "jpg".
            - ignore_cached: always re-render the tile, whether it's in the cache or not.
        
            This is the main entry point, after site configuration has been loaded
            and individual tiles need to be rendered.
        """
        start_time = time()
        
        mimetype, format = self.getTypeByExtension(extension)

        # default response values
        status_code = 200
        headers = Headers([('Content-Type', mimetype)])
        body = None

        cache = self.config.cache

        if not ignore_cached:
            # Start by checking for a tile in the cache.
            try:
                body = cache.read(self, coord, format)
            except TheTileLeftANote, e:
                headers = e.headers
                status_code = e.status_code
                body = e.content

                if e.emit_content_type:
                    headers.setdefault('Content-Type', mimetype)

            tile_from = 'cache'
Exemplo n.º 10
0
class Response(threading.local):
    """ Represents a single response using thread-local namespace. """

    def bind(self, app):
        """ Clears old data and creates a brand new Response object """
        self._COOKIES = None
        self.status = 200
        self.header_list = []
        self.header = HeaderWrapper(self.header_list)
        self.charset = 'UTF-8'
        self.content_type = 'text/html; charset=UTF-8'
        self.error = None
        self.app = app

    def add_header(self, key, value):
        self.header.add_header(key.title(), str(value))

    def wsgiheaders(self):
        ''' Returns a wsgi conform list of header/value pairs '''
        for c in self.COOKIES.values():
            self.add_header('Set-Cookie', c.OutputString())
        return self.header_list

    @property
    def COOKIES(self):
        if not self._COOKIES:
            self._COOKIES = SimpleCookie()
        return self._COOKIES

    def set_cookie(self, key, value, **kargs):
        """
        Sets a Cookie. Optional settings:
        expires, path, comment, domain, max-age, secure, version, httponly
        """
        if not isinstance(value, basestring):
            sec = self.app.config['securecookie.key']
            value = cookie_encode(value, sec)
        self.COOKIES[key] = value
        for k, v in kargs.iteritems():
            self.COOKIES[key][k] = v

    def get_content_type(self):
        """ Get the current 'Content-Type' header. """
        return self.header['Content-Type']
        
    def set_content_type(self, value):
        if 'charset=' in value:
            self.charset = value.split('charset=')[-1].split(';')[0].strip()
        self.header['Content-Type'] = value

    content_type = property(get_content_type, set_content_type, None,
                            get_content_type.__doc__)
Exemplo n.º 11
0
 def get_alternatives(base_headers, files):
     # Sort by size so that the smallest compressed alternative matches first
     alternatives = []
     files_by_size = sorted(files.items(), key=lambda i: i[1].stat.st_size)
     for encoding, file_entry in files_by_size:
         headers = Headers(base_headers.items())
         headers['Content-Length'] = str(file_entry.stat.st_size)
         if encoding:
             headers['Content-Encoding'] = encoding
             encoding_re = re.compile(r'\b%s\b' % encoding)
         else:
             encoding_re = re.compile('')
         alternatives.append((encoding_re, file_entry.path, headers.items()))
     return alternatives
Exemplo n.º 12
0
    def signal_land_or_sea(self, body, layer, coord, format):
        if body:
            md5sum = self.md5sum(body)
            second_md5sum = self.md5sum(Disk.read(self, self.second, coord, format))
            
            headers = Headers([('Access-Control-Expose-Headers', 'X-Land-Or-Sea')])
            headers.setdefault('X-Land-Or-Sea', '0')
            
            if second_md5sum and md5sum == second_md5sum:
                if md5sum == self.land_md5:
                    headers['X-Land-Or-Sea'] = '1'
                elif md5sum == self.sea_md5:
                    headers['X-Land-Or-Sea'] = '2'

            raise TheTileLeftANote(content=body, headers=headers)
Exemplo n.º 13
0
    def __call__(self, environ, start_response):
        key_morsel = Cookie(environ.get("HTTP_COOKIE", "")).get(self.toggle_key)
        # useful vars
        query = query_str2dict(environ.get("QUERY_STRING"))
        enable_by_cookie = key_morsel.value == self.enable_value if key_morsel else False
        enable_by_query = query.get(self.toggle_key) == self.enable_value
        # pop toggle_key from query dic to avoid case: '?_profile=on&_profile='
        disable = query.pop(self.toggle_key, None) == ""  # only can be disabled by query
        enable = not disable and (enable_by_query or enable_by_cookie)

        run_app, resp_body, saved_ss_args = self._intercept_call()

        # processing cookies and queries
        so = query.pop(self.SIMPLE_OUTPUT_TOGGLE_KEY, None)
        if so is not None:
            self.simple_output = so == "True"
        cookie_to_set = None
        if enable_by_query and not enable_by_cookie:
            cookie_to_set = "%s=%s; Path=/; HttpOnly" % (self.toggle_key, self.enable_value)
        elif disable:
            cookie_to_set = "%s=; Path=/; Max-Age=1; HttpOnly" % self.toggle_key

        if enable:
            start = time.time()
            profile = Profile()
            profile.runcall(run_app, environ)  # here we call the WSGI app
            elapsed = time.time() - start
        else:
            profile = elapsed = None  # for annoying IDE
            run_app(environ)

        status, headers = saved_ss_args[:2]
        headers_dic = Headers(headers)
        if cookie_to_set:
            headers_dic.add_header("Set-Cookie", cookie_to_set)

        # insert result into response
        content_type = headers_dic.get("Content-Type", "")
        if enable and status.startswith("200") and content_type.startswith("text/html"):
            environ["QUERY_STRING"] = dict2query_str(query)

            matched = _find_charset.match(content_type)
            encoding = matched.group(1) if matched else "ascii"
            rendered = self.render_result(profile, elapsed, environ).encode(encoding, "replace")
            resp_body = [insert_into_body(rendered, b"".join(resp_body))]
            headers_dic["Content-Length"] = str(len(resp_body[0]))
        start_response(status, headers, saved_ss_args[2] if len(saved_ss_args) == 3 else None)
        return resp_body
Exemplo n.º 14
0
 def get_static_file(self, path, url, stat_cache=None):
     # Optimization: bail early if file does not exist
     if stat_cache is None and not os.path.exists(path):
         raise MissingFileError(path)
     headers = Headers([])
     self.add_mime_headers(headers, path, url)
     self.add_cache_headers(headers, path, url)
     if self.allow_all_origins:
         headers['Access-Control-Allow-Origin'] = '*'
     if self.add_headers_function:
         self.add_headers_function(headers, path, url)
     return StaticFile(
             path, headers.items(),
             stat_cache=stat_cache,
             encodings={
               'gzip': path + '.gz', 'br': path + '.br'})
Exemplo n.º 15
0
def _parse_headers(environ):
    """
    Parse the environmental variables, looking for HTTP request headers.
    :param environ: environmental variables
    :type environ: dict
    :return: request headers
    :rtype: dict
    """
    headers = Headers([])
    for key, value in environ.items():
        match = _HTTP_HEADER_REGEX.match(key)
        if match is None:
            continue
        name = _normalize_header_name(match.group(0))
        headers.add_header(name, value)
    return headers
Exemplo n.º 16
0
    def __init__(self, path, is_immutable, guess_type=mimetypes.guess_type, **config):
        self.path = path
        stat = os.stat(path)
        self.mtime_tuple = gmtime(stat.st_mtime)
        mimetype, encoding = guess_type(path)
        mimetype = mimetype or 'application/octet-stream'
        charset = self.get_charset(mimetype)
        params = {'charset': charset} if charset else {}
        self.headers = Headers([
            ('Last-Modified', formatdate(stat.st_mtime, usegmt=True)),
            ('Content-Length', str(stat.st_size)),
        ])
        self.headers.add_header('Content-Type', str(mimetype), **params)
        if encoding:
            self.headers['Content-Encoding'] = encoding

        max_age = self.FOREVER if is_immutable else config['max_age']
        if max_age is not None:
            self.headers['Cache-Control'] = 'public, max-age=%s' % max_age

        if config['allow_all_origins']:
            self.headers['Access-Control-Allow-Origin'] = '*'

        gzip_path = path + self.GZIP_SUFFIX
        if os.path.isfile(gzip_path):
            self.gzip_path = gzip_path
            self.headers['Vary'] = 'Accept-Encoding'
            # Copy the headers and add the appropriate encoding and length
            self.gzip_headers = Headers(self.headers.items())
            self.gzip_headers['Content-Encoding'] = 'gzip'
            self.gzip_headers['Content-Length'] = str(os.stat(gzip_path).st_size)
        else:
            self.gzip_path = self.gzip_headers = None
Exemplo n.º 17
0
 def bind(self):
     """ Clears old data and creates a brand new Response object """
     self._COOKIES = None
     self.status = 200
     self.header_list = []
     self.header = HeaderWrapper(self.header_list)
     self.content_type = 'text/html'
     self.error = None
Exemplo n.º 18
0
    def __init__(self, body=b'', status=None, headers=None):
        self.headers = Headers()
        self._body = body
        self._status_code = status or self.default_status
        self._cookies = SimpleCookie()

        if headers:
            for name, value in headers.items():
                self.headers.add_header(name, value)
Exemplo n.º 19
0
    def newapp(environ, start_response):
        body = app(environ, capture)
        status = resp['status']
        headers = Headers(resp['headers'])

        already = 'Content-Encoding' in headers
        accepted = 'gzip' in environ.get('HTTP_ACCEPT_ENCODING', '')
        if not accepted or already:
            # no compress
            start_response(status, list(headers.items()))
            return body

        content = gzip.compress(b''.join(body))
        if hasattr(body, 'close'):
            body.close()
        headers['Content-Encoding'] = 'gzip'
        start_response(status, list(headers.items()))
        return [content]
Exemplo n.º 20
0
class Response(threading.local):
    """ Represents a single response using thread-local namespace. """

    def bind(self):
        """ Clears old data and creates a brand new Response object """
        self._COOKIES = None
        self.status = 200
        self.header_list = []
        self.header = HeaderWrapper(self.header_list)
        self.content_type = "text/html"
        self.error = None
        self.charset = "utf8"

    def wsgiheaders(self):
        """ Returns a wsgi conform list of header/value pairs """
        for c in self.COOKIES.itervalues():
            self.header.add_header("Set-Cookie", c.OutputString())
        return [(h.title(), str(v)) for h, v in self.header.items()]

    @property
    def COOKIES(self):
        if not self._COOKIES:
            self._COOKIES = SimpleCookie()
        return self._COOKIES

    def set_cookie(self, key, value, **kargs):
        """
        Sets a Cookie. Optional settings:
        expires, path, comment, domain, max-age, secure, version, httponly
        """
        self.COOKIES[key] = value
        for k, v in kargs.iteritems():
            self.COOKIES[key][k] = v

    def get_content_type(self):
        """ Get the current 'Content-Type' header. """
        return self.header["Content-Type"]

    def set_content_type(self, value):
        if "charset=" in value:
            self.charset = value.split("charset=")[-1].split(";")[0].strip()
        self.header["Content-Type"] = value

    content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__)
Exemplo n.º 21
0
Arquivo: bottle.py Projeto: apg/bottle
 def bind(self, app):
     """ Clears old data and creates a brand new Response object """
     self._COOKIES = None
     self.status = 200
     self.header_list = []
     self.header = HeaderWrapper(self.header_list)
     self.charset = "UTF-8"
     self.content_type = "text/html; charset=UTF-8"
     self.error = None
     self.app = app
Exemplo n.º 22
0
    def newapp(environ, start_response):
        raw = app(environ, capture)
        status = resp['status']
        headers = Headers(resp['headers'])

        if (not 'Last-Modified' in headers
            or not environ.get('HTTP_IF_MODIFIED_SINCE')):
            start_response(status, list(headers.items()))
            return raw

        last_m = eutils.parsedate(headers['Last-Modified'])
        since_m = eutils.parsedate(environ['HTTP_IF_MODIFIED_SINCE'])
        if since_m < last_m:
            start_response(status, list(headers.items()))
            return raw
        else:
            start_response('304 Not Modified', list(headers.items()))
            if hasattr(raw, 'close'):
                raw.close()
            return [b'']
Exemplo n.º 23
0
def board_app(env, resp):
    path = env['PATH_INFO']
    m = board_re.match(path)
    board = m.group(1)
    message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja'))

    headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')])
    resp("200 OK", headers.items())

    html = [
        '<!DOCTYPE html>',
        '<html><head>',
        '<meta http-equiv="content-type" content="text/html; charset=Shift_JIS">',
        '<title>%s - %s</title>' % (message['logo'], message['description']),
        '<meta name="description" content="%s - %s">' % (message['logo'], message['description']),
        '</head><body>',
        '<h1>%s - %s</h1>' % (message['logo'], message['description']),
        '</body></html>',
    ]
    return ((c + '\n').encode('sjis', 'ignore') for c in html)
Exemplo n.º 24
0
 def finish_header(self):
     self.file = BytesIO()
     self.headers = Headers(self.headerlist)
     cdis = self.headers.get('Content-Disposition', '')
     ctype = self.headers.get('Content-Type', '')
     if not cdis:
         raise MultipartError('Content-Disposition header is missing.')
     self.disposition, self.options = parse_options_header(cdis)
     self.name = self.options.get('name')
     self.filename = self.options.get('filename')
     self.content_type, options = parse_options_header(ctype)
     self.charset = options.get('charset') or self.charset
     self.content_length = int(self.headers.get('Content-Length', '-1'))
Exemplo n.º 25
0
class Response(threading.local):
    """ Represents a single response using thread-local namespace. """

    def bind(self):
        """ Clears old data and creates a brand new Response object """
        self._COOKIES = None
        self.status = 200
        self.header_list = []
        self.header = HeaderWrapper(self.header_list)
        self.content_type = 'text/html'
        self.error = None

    def wsgiheaders(self):
        ''' Returns a wsgi conform list of header/value pairs '''
        for c in self.COOKIES.itervalues():
            self.header.add_header('Set-Cookie', c.OutputString())
        return [(h.title(), str(v)) for h, v in self.header_list]

    @property
    def COOKIES(self):
        if not self._COOKIES:
            self._COOKIES = SimpleCookie()
        return self._COOKIES

    def set_cookie(self, key, value, **kargs):
        """ Sets a Cookie. Optional settings: expires, path, comment, domain, max-age, secure, version, httponly """
        self.COOKIES[key] = value
        for k in kargs:
            self.COOKIES[key][k] = kargs[k]

    def get_content_type(self):
        '''Gives access to the 'Content-Type' header and defaults to 'text/html'.'''
        return self.header['Content-Type']
        
    def set_content_type(self, value):
        self.header['Content-Type'] = value
        
    content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__)
Exemplo n.º 26
0
 def finish_header(self):
     self.file = BytesIO()
     self.headers = Headers(self.headerlist)
     cdis = self.headers.get("Content-Disposition", "")
     ctype = self.headers.get("Content-Type", "")
     clen = self.headers.get("Content-Length", "-1")
     if not cdis:
         raise MultipartError("Content-Disposition header is missing.")
     self.disposition, self.options = parse_options_header(cdis)
     self.name = self.options.get("name")
     self.filename = self.options.get("filename")
     self.content_type, options = parse_options_header(ctype)
     self.charset = options.get("charset") or self.charset
     self.content_length = int(self.headers.get("Content-Length", "-1"))
Exemplo n.º 27
0
class Response(object):
    """
    Wrapper class around the start_response and return iterable of the WSGI
    protocol.

    :ivar status: HTTP status of the response
    :type status: int
    :ivar headers: response headers
    :type headers: dict
    :ivar cookies: cookies to set
    :type cookies: dict
    """

    def __init__(self):
        self.status = None
        self.__header_list = []
        self.headers = Headers(self.__header_list)
        self.headers['date'] = rfc1123_date()
        self.cookies = SimpleCookie()

    def start_response_args(self, body):
        """
        Return a status string and a list of headers for this response,
        appropriate as arguments to the start_response function.
        :param body: response body
        :type body: str
        :return: tuple of status string and list of header tuples
        :rtype: (str, list)
        """
        if self.status is None:
            raise RuntimeError('response status was not set')
        self.headers['content-length'] = str(len(body))
        for cookie in self.cookies:
            self.headers.add_header('set-cookie', cookie.output(header=''))
        status_str = '%d %s' % (self.status, httplib.responses[self.status])
        return (status_str, self.__header_list)
Exemplo n.º 28
0
 def finish_header(self):
     self.file = BytesIO()
     self.headers = Headers(self.headerlist)
     cdis = self.headers.get('Content-Disposition','')
     ctype = self.headers.get('Content-Type','')
     clen = self.headers.get('Content-Length','-1')
     if not cdis:
         raise MultipartError('Content-Disposition header is missing.')
     self.disposition, self.options = parse_options_header(cdis)
     self.name = self.options.get('name')
     self.filename = self.options.get('filename')
     self.content_type, options = parse_options_header(ctype)
     self.charset = options.get('charset') or self.charset
     self.content_length = int(self.headers.get('Content-Length','-1'))
     self.content_transfer_encoding = self.headers.get('Content-Transfer-Encoding')
     if self.content_transfer_encoding not in [None, 'base64', 'quoted-printable']:
         raise MultipartError('invalid Content-Transfer-Encoding')
Exemplo n.º 29
0
    def testExtras(self):
        h = Headers([])
        self.assertEqual(str(h), "\r\n")

        h.add_header("foo", "bar", baz="spam")
        self.assertEqual(h["foo"], 'bar; baz="spam"')
        self.assertEqual(str(h), 'foo: bar; baz="spam"\r\n\r\n')

        h.add_header("Foo", "bar", cheese=None)
        self.assertEqual(h.get_all("foo"), ['bar; baz="spam"', "bar; cheese"])

        self.assertEqual(str(h), 'foo: bar; baz="spam"\r\n' "Foo: bar; cheese\r\n" "\r\n")
Exemplo n.º 30
0
    def testExtras(self):
        h = Headers([])
        self.assertEqual(str(h),'\r\n')

        h.add_header('foo','bar',baz="spam")
        self.assertEqual(h['foo'], 'bar; baz="spam"')
        self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n')

        h.add_header('Foo','bar',cheese=None)
        self.assertEqual(h.get_all('foo'),
            ['bar; baz="spam"', 'bar; cheese'])

        self.assertEqual(str(h),
            'foo: bar; baz="spam"\r\n'
            'Foo: bar; cheese\r\n'
            '\r\n'
        )
Exemplo n.º 31
0
class MultipartPart(object):
    def __init__(self,
                 buffer_size=2**16,
                 memfile_limit=2**18,
                 charset="latin1"):
        self.headerlist = []
        self.headers = None
        self.file = False
        self.size = 0
        self._buf = b""
        self.disposition = None
        self.name = None
        self.filename = None
        self.content_type = None
        self.charset = charset
        self.memfile_limit = memfile_limit
        self.buffer_size = buffer_size

    def feed(self, line, nl=""):
        if self.file:
            return self.write_body(line, nl)

        return self.write_header(line, nl)

    def write_header(self, line, nl):
        line = line.decode(self.charset)

        if not nl:
            raise MultipartError("Unexpected end of line in header.")

        if not line.strip():  # blank line -> end of header segment
            self.finish_header()
        elif line[0] in " \t" and self.headerlist:
            name, value = self.headerlist.pop()
            self.headerlist.append((name, value + line.strip()))
        else:
            if ":" not in line:
                raise MultipartError("Syntax error in header: No colon.")

            name, value = line.split(":", 1)
            self.headerlist.append((name.strip(), value.strip()))

    def write_body(self, line, nl):
        if not line and not nl:
            return  # This does not even flush the buffer

        self.size += len(line) + len(self._buf)
        self.file.write(self._buf + line)
        self._buf = nl

        if self.content_length > 0 and self.size > self.content_length:
            raise MultipartError("Size of body exceeds Content-Length header.")

        if self.size > self.memfile_limit and isinstance(self.file, BytesIO):
            # TODO: What about non-file uploads that exceed the memfile_limit?
            self.file, old = TemporaryFile(mode="w+b"), self.file
            old.seek(0)
            copy_file(old, self.file, self.size, self.buffer_size)

    def finish_header(self):
        self.file = BytesIO()
        self.headers = Headers(self.headerlist)
        content_disposition = self.headers.get("Content-Disposition", "")
        content_type = self.headers.get("Content-Type", "")

        if not content_disposition:
            raise MultipartError("Content-Disposition header is missing.")

        self.disposition, self.options = parse_options_header(
            content_disposition)
        self.name = self.options.get("name")
        self.filename = self.options.get("filename")
        self.content_type, options = parse_options_header(content_type)
        self.charset = options.get("charset") or self.charset
        self.content_length = int(self.headers.get("Content-Length", "-1"))

    def is_buffered(self):
        """ Return true if the data is fully buffered in memory."""
        return isinstance(self.file, BytesIO)

    @property
    def value(self):
        """ Data decoded with the specified charset """

        return self.raw.decode(self.charset)

    @property
    def raw(self):
        """ Data without decoding """
        pos = self.file.tell()
        self.file.seek(0)

        try:
            val = self.file.read()
        except IOError:
            raise
        finally:
            self.file.seek(pos)

        return val

    def save_as(self, path):
        with open(path, "wb") as fp:
            pos = self.file.tell()

            try:
                self.file.seek(0)
                size = copy_file(self.file, fp)
            finally:
                self.file.seek(pos)

        return size

    def close(self):
        if self.file:
            self.file.close()
            self.file = False
 def _start_response(self, status: str, response_headers: List[Any]):
     self._status = status
     self._headers = Headers(response_headers)  # type: ignore
     self._status_code = int(self._status.split(' ')[0])  # 200 OK
Exemplo n.º 33
0
class SimDistribHandler(object):
    def __init__(self, archive, wsgienv, start_resp):
        self.arch = archive
        self._env = wsgienv
        self._start = start_resp
        self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
        self._hdr = Headers([])
        self._code = 0
        self._msg = "unknown status"

    def send_error(self, code, message):
        status = "{0} {1}".format(str(code), message)
        self._start(status, [], sys.exc_info())
        return []

    def add_header(self, name, value):
        self._hdr.add_header(name, value)

    def set_response(self, code, message):
        self._code = code
        self._msg = message

    def end_headers(self):
        status = "{0} {1}".format(str(self._code), self._msg)
        self._start(status, self._hdr.items())

    def handle(self, env, start_resp):
        meth_handler = 'do_' + self._meth

        path = self._env.get('PATH_INFO', '/')[1:]
        params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))

        if hasattr(self, meth_handler):
            return getattr(self, meth_handler)(path, params)
        else:
            return self.send_error(
                403, self._meth + " not supported on this resource")

    def do_HEAD(self, path, params=None, forhead=False):
        return self.do_GET(path, params, True)

    def do_GET(self, path, params=None, forhead=False):
        aid = None
        vers = None
        path = path.strip('/')
        if path.startswith("od/ds/"):
            path = path[len("od/ds/"):]
        print("processing " + path)

        # refresh the archive
        self.arch.loadinfo()

        if not path:
            try:
                out = json.dumps(self.arch.aipids) + '\n'
            except Exception as ex:
                return self.send_error(500, "Internal error")

            self.set_response(200, "AIP Identifiers")
            self.add_header('Content-Type', 'application/json')
            self.add_header('Content-Length', str(len(out)))
            self.end_headers()
            if forhead:
                return []
            return [out]

        elif path.startswith("_aip/"):
            # requesting a bag file
            path = path[len("_aip/"):].strip('/')
            filepath = os.path.join(self.arch.dir, path)
            if os.path.isfile(filepath):
                self.set_response(200, "Bag file found")
                self.add_header('Content-Type', "application/zip")
                self.end_headers()
                if forhead:
                    return []
                return self.iter_file(filepath)
            else:
                return self.send_error(404, "bag file does not exist")

        elif '/' in path:
            parts = path.split('/', 1)
            aid = parts[0]
            path = (len(parts) > 1 and parts[1]) or ''
            print("accessing " + aid)

        elif path:
            aid = path
            path = ''

        else:
            return self.send_error(404, "resource does not exist")

        # path-info is now captured as aid and path
        if aid not in self.arch._aips:
            return self.send_error(404, "resource does not exist")

        if not path:
            self.set_response(200, "AIP Identifier exists")
            self.add_header('Content-Type', 'application/json')
            self.add_header('Content-Length', str(len(aid) + 4))
            self.end_headers()
            if forhead:
                return []
            return ['["' + aid + '"]']

        elif path == "_aip":
            try:
                out = json.dumps(self.arch.list_bags(aid)) + '\n'
            except Exception, ex:
                return self.send_error(500, "Internal error")

            self.set_response(200, "All bags for ID")
            self.add_header('Content-Type', 'application/json')
            self.add_header('Content-Length', str(len(out)))
            self.end_headers()
            if forhead:
                return []
            return [out]

        elif path == "_aip/_head":
            try:
                out = self.arch.head_for(aid)
                if out:
                    out = json.dumps(out) + '\n'
            except Exception, ex:
                print(
                    "Failed to create JSON output for head bag, aid={0}: {2}".
                    format(aid, vers, str(ex)))
                return self.send_error(500, "Internal error")
Exemplo n.º 34
0
 def __init__(self, path):
     self.path = path
     self.headers = Headers([])
Exemplo n.º 35
0
class SimIngestHandler(object):
    def __init__(self, wsgienv, start_resp):
        self._env = wsgienv
        self._start = start_resp
        self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
        self._hdr = Headers([])
        self._code = 0
        self._msg = "unknown status"
        self._auth = (authmeth, authkey)

    def send_error(self, code, message):
        status = "{0} {1}".format(str(code), message)
        self._start(status, [], sys.exc_info())
        return []

    def add_header(self, name, value):
        self._hdr.add_header(name, value)

    def set_response(self, code, message):
        self._code = code
        self._msg = message

    def end_headers(self):
        status = "{0} {1}".format(str(self._code), self._msg)
        self._start(status, self._hdr.items())

    def handle(self, env, start_resp):
        meth_handler = 'do_' + self._meth

        path = self._env.get('PATH_INFO', '/')[1:]
        params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
        print("AUTH METHOD: %s" % self._auth[0], file=sys.stderr)
        if not self.authorize():
            return self.send_unauthorized()

        if hasattr(self, meth_handler):
            return getattr(self, meth_handler)(path, params)
        else:
            return self.send_error(
                403, self._meth + " not supported on this resource")

    def authorize(self):
        if self._auth[0] == 'header':
            return self.authorize_via_headertoken()
        else:
            return self.authorize_via_queryparam()

    def authorize_via_queryparam(self):
        params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
        auths = params.get('auth', [])
        if self._auth[1]:
            # match the last value provided
            return len(auths) > 0 and self._auth[1] == auths[-1]
        if len(auths) > 0:
            log.warn(
                "Authorization key provided, but none has been configured")
        return len(auths) == 0

    def authorize_via_headertoken(self):
        authhdr = self._env.get('HTTP_AUTHORIZATION', "")
        print("Request HTTP_AUTHORIZATION: %s" % authhdr, file=sys.stderr)
        parts = authhdr.split()
        if self._auth[1]:
            return len(parts) > 1 and parts[0] == "Bearer" and \
                self._auth[1] == parts[1]
        if authhdr:
            log.warn(
                "Authorization key provided, but none has been configured")
        return authhdr == ""

    def send_unauthorized(self):
        self.set_response(401, "Not authorized")
        if self._auth[0] == 'header':
            self.add_header('WWW-Authenticate', 'Bearer')
        self.end_headers()
        return []

    def do_GET(self, path, params=None):
        path = path.strip('/')
        if not path:
            try:
                out = json.dumps(["nerdm", "invalid"]) + '\n'
            except Exception, ex:
                return self.send_error(500, "Internal error")

            self.set_response(200, "Supported Record Types")
            self.add_header('Content-Type', 'application/json')
            self.end_headers()
            return [out]
        elif path in "nerdm invalid".split():
            self.set_response(200, "Service is ready")
            self.add_header('Content-Type', 'application/json')
            self.end_headers()
            return ["Service ready\n"]
Exemplo n.º 36
0
def test_auth_ok(static_auth):
    request = mock.Mock()
    request.headers = Headers([("authorization", "Bearer test")])
    assert static_auth.is_authorized(request)
Exemplo n.º 37
0

TLDLESS_NETLOC_RESOLVER = {
    'ketab': ketabir_scr,
    'noorlib': noorlib_scr,
    'noormags': noormags_scr,
    'web.archive': waybackmachine_scr,
    'web-beta.archive': waybackmachine_scr,
    'books.google.co': googlebooks_scr,
    'books.google.com': googlebooks_scr,
    'books.google': googlebooks_scr,
    'google': google_encrypted_scr,
    'encrypted.google': google_encrypted_scr,
}.get

RESPONSE_HEADERS = Headers([('Content-Type', 'text/html; charset=UTF-8')])

getLogger('requests').setLevel(WARNING)
getLogger('langid').setLevel(WARNING)


def get_root_logger():
    custom_logger = getLogger()
    custom_logger.setLevel(INFO)
    srcdir = dirname(__file__)
    handler = RotatingFileHandler(filename=f'{srcdir}/citer.log',
                                  mode='a',
                                  maxBytes=20000,
                                  backupCount=0,
                                  encoding='utf-8',
                                  delay=0)
Exemplo n.º 38
0
        'application/x-jpg': '.jpg',
        'application/x-bmp': '.bmp',
        'application/msword': '.doc',
        '': '',
    }
    return dict[type] if type in dict.keys() else ''


DEFAULT_MAX_THREAD = 5
DEFAULT_MAX_CONNECTIONS = 16

HEADERS_CHROME = Headers([
    ('User-Agent',
     'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
     ),
    ('Accept',
     'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8'
     ), ('Accept-Encoding', 'gzip, deflate, br'),
    ('Accept-Language', 'zh-CN,zh;q=0.9')
])


class UrlPool(Packer, object):
    def __init__(self,
                 parent,
                 max_retry=-1,
                 max_conn=DEFAULT_MAX_CONNECTIONS,
                 max_speed=-1):
        self.parent = parent
        self.list = []
        self.dict = {}
Exemplo n.º 39
0
    def newapp(environ, start_response):
        raw = app(environ, capture)
        status = resp['status']
        headers = Headers(resp['headers'])

        headers.setdefault('Accept-Range', 'bytes')
        range = environ.get('HTTP_RANGE')

        if (range is None or ',' in range  # not deal with multi-part range
                or not status.startswith('2')):  # not success status
            start_response(status, list(headers.items()))
            return raw

        def error_416():
            start_response('416 Requested Range Not Satisfiable',
                           list(headers.items()))
            if hasattr(raw, 'close'):
                raw.close()
            return [b'']

        m = re.match(r'bytes=([0-9]+)?-([0-9]+)?', range)
        if not m or (not m.group(1) and not m.group(2)):
            return error_416()

        content = b''.join(raw)
        begin = int(m.group(1)) if m.group(1) else None
        end = int(m.group(2)) if m.group(2) else None

        # because 0 is False
        has_begin = begin is not None
        has_end = end is not None

        if (has_begin and has_end) and end < begin:
            return error_416()
        if has_end and len(content) <= end:
            return error_416()
        if has_begin and len(content) <= begin:
            return error_416()

        if has_begin and has_end:
            # bytes=begin-end
            c_range = 'bytes {}-{}/{}'.format(begin, end, len(content))
            body = content[begin:end + 1]

        elif has_begin:
            # bytes=begin-
            c_range = 'bytes {}-{}/{}'.format(begin,
                                              len(content) - 1, len(content))
            body = content[begin:]

        else:
            # bytes=-end
            c_range = 'bytes {}-{}/{}'.format(
                len(content) - end,
                len(content) - 1, len(content))
            body = content[len(content) - end:]

        headers['Content-Range'] = c_range
        start_response('206 Partial Content', list(headers.items()))
        if hasattr(raw, 'close'):
            raw.close()
        return [body]
Exemplo n.º 40
0
def getPreview(layer):
    """ Get a type string and dynamic map viewer HTML for a given layer.
    """
    return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)
Exemplo n.º 41
0
    def testMappingInterface(self):
        test = [('x','y')]
        self.assertEqual(len(Headers()), 0)
        self.assertEqual(len(Headers([])),0)
        self.assertEqual(len(Headers(test[:])),1)
        self.assertEqual(Headers(test[:]).keys(), ['x'])
        self.assertEqual(Headers(test[:]).values(), ['y'])
        self.assertEqual(Headers(test[:]).items(), test)
        self.assertIsNot(Headers(test).items(), test)  # must be copy!

        h = Headers()
        del h['foo']   # should not raise an error

        h['Foo'] = 'bar'
        for m in h.__contains__, h.get, h.get_all, h.__getitem__:
            self.assertTrue(m('foo'))
            self.assertTrue(m('Foo'))
            self.assertTrue(m('FOO'))
            self.assertFalse(m('bar'))

        self.assertEqual(h['foo'],'bar')
        h['foo'] = 'baz'
        self.assertEqual(h['FOO'],'baz')
        self.assertEqual(h.get_all('foo'),['baz'])

        self.assertEqual(h.get("foo","whee"), "baz")
        self.assertEqual(h.get("zoo","whee"), "whee")
        self.assertEqual(h.setdefault("foo","whee"), "baz")
        self.assertEqual(h.setdefault("zoo","whee"), "whee")
        self.assertEqual(h["foo"],"baz")
        self.assertEqual(h["zoo"],"whee")
Exemplo n.º 42
0
def requestHandler2(config_hint, path_info, query_string=None, script_name=''):
    """ Generate a set of headers and response body for a given request.

        TODO: Replace requestHandler() with this function in TileStache 2.0.0.

        Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png").

        Config_hint parameter can be a path string for a JSON configuration file
        or a configuration object with 'cache', 'layers', and 'dirpath' properties.

        Query string is optional, currently used for JSON callbacks.

        Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html.
    """
    headers = Headers([])

    try:
        # ensure that path_info is at least a single "/"
        path_info = '/' + (path_info or '').lstrip('/')

        layer = requestLayer(config_hint, path_info)
        query = parse_qs(query_string or '')
        try:
            callback = query['callback'][0]
        except KeyError:
            callback = None

        #
        # Special case for index page.
        #
        if path_info == '/':
            mimetype, content = getattr(
                layer.config, 'index',
                ('text/plain', 'TileStache says hello.'))
            return 200, Headers([('Content-Type', mimetype)]), content

        coord, extension = splitPathInfo(path_info)[1:]

        if extension == 'html' and coord is None:
            status_code, headers, content = getPreview(layer)

        elif extension.lower() in layer.redirects:
            other_extension = layer.redirects[extension.lower()]

            redirect_uri = script_name
            redirect_uri += mergePathInfo(layer.name(), coord, other_extension)

            if query_string:
                redirect_uri += '?' + query_string

            headers['Location'] = redirect_uri
            headers['Content-Type'] = 'text/plain'

            return 302, headers, 'You are being redirected to %s\n' % redirect_uri

        else:
            status_code, headers, content = layer.getTileResponse(
                coord, extension)

        if layer.allowed_origin:
            headers.setdefault('Access-Control-Allow-Origin',
                               layer.allowed_origin)

        if callback and 'json' in headers['Content-Type']:
            headers['Content-Type'] = 'application/javascript; charset=utf-8'
            content = '%s(%s)' % (callback, content)

        if layer.max_cache_age is not None:
            expires = datetime.utcnow() + timedelta(
                seconds=layer.max_cache_age)
            headers.setdefault('Expires',
                               expires.strftime('%a, %d %b %Y %H:%M:%S GMT'))
            headers.setdefault('Cache-Control',
                               'public, max-age=%d' % layer.max_cache_age)

    except Core.KnownUnknown as e:
        out = StringIO()

        print('Known unknown!', file=out)
        print(e, file=out)
        print('', file=out)
        print('\n'.join(Core._rummy()), file=out)

        headers['Content-Type'] = 'text/plain'
        status_code, content = 500, out.getvalue().encode('ascii')

    return status_code, headers, content
Exemplo n.º 43
0
 def __getitem__(self, name):
     ret = Headers.__getitem__(self, name)
     if ret is None:
         return ''
     else:
         return str(ret)
Exemplo n.º 44
0
    def __call__(self, environ, start_response):
        '''Main function for handling a single request. Follows the
		WSGI API.

		@param environ: dictionary with environment variables for the
		request and some special variables. See the PEP for expected
		variables.

		@param start_response: a function that can be called to set the
		http response and headers. For example::

			start_response(200, [('Content-Type', 'text/plain')])

		@returns: the html page content as a list of lines
		'''
        headerlist = []
        headers = Headers(headerlist)
        path = environ.get('PATH_INFO', '/')
        try:
            methods = ('GET', 'HEAD')
            if not environ['REQUEST_METHOD'] in methods:
                raise WWWError('405', headers=[('Allow', ', '.join(methods))])

            # cleanup path
            #~ print 'INPUT', path
            path = path.replace('\\', '/')  # make it windows save
            isdir = path.endswith('/')
            parts = [p for p in path.split('/') if p and not p == '.']
            if [p for p in parts if p.startswith('.')]:
                # exclude .. and all hidden files from possible paths
                raise PathNotValidError()
            path = '/' + '/'.join(parts)
            if isdir and not path == '/': path += '/'
            #~ print 'PATH', path

            if not path:
                path = '/'
            elif path == '/favicon.ico':
                path = '/+resources/favicon.ico'
            else:
                path = urllib.unquote(path)

            if path == '/':
                headers.add_header('Content-Type',
                                   'text/html',
                                   charset='utf-8')
                content = self.render_index()
            elif path.startswith('/+docs/'):
                dir = self.notebook.document_root
                if not dir:
                    raise PageNotFoundError(path)
                file = dir.file(path[7:])
                content = [file.raw()]
                # Will raise FileNotFound when file does not exist
                headers['Content-Type'] = file.get_mimetype()
            elif path.startswith('/+file/'):
                file = self.notebook.dir.file(path[7:])
                # TODO: need abstraction for getting file from top level dir ?
                content = [file.raw()]
                # Will raise FileNotFound when file does not exist
                headers['Content-Type'] = file.get_mimetype()
            elif path.startswith('/+resources/'):
                if self.template.resources_dir:
                    file = self.template.resources_dir.file(path[12:])
                    if not file.exists():
                        file = data_file('pixmaps/%s' % path[12:])
                else:
                    file = data_file('pixmaps/%s' % path[12:])

                if file:
                    content = [file.raw()]
                    # Will raise FileNotFound when file does not exist
                    headers['Content-Type'] = file.get_mimetype()
                else:
                    raise PageNotFoundError(path)
            else:
                # Must be a page or a namespace (html file or directory path)
                headers.add_header('Content-Type',
                                   'text/html',
                                   charset='utf-8')
                if path.endswith('.html'):
                    pagename = path[:-5].replace('/', ':')
                elif path.endswith('/'):
                    pagename = path[:-1].replace('/', ':')
                else:
                    raise PageNotFoundError(path)

                path = self.notebook.resolve_path(pagename)
                page = self.notebook.get_page(path)
                if page.hascontent:
                    content = self.render_page(page)
                elif page.haschildren:
                    content = self.render_index(page)
                else:
                    raise PageNotFoundError(page)
        except Exception, error:
            headerlist = []
            headers = Headers(headerlist)
            headers.add_header('Content-Type', 'text/plain', charset='utf-8')
            if isinstance(error, (WWWError, FileNotFoundError)):
                logger.error(error.msg)
                if isinstance(error, FileNotFoundError):
                    error = PageNotFoundError(path)
                    # show url path instead of file path
                if error.headers:
                    for key, value in error.headers:
                        headers.add_header(key, value)
                start_response(error.status, headerlist)
                content = unicode(error).splitlines(True)
            # TODO also handle template errors as special here
            else:
                # Unexpected error - maybe a bug, do not expose output on bugs
                # to the outside world
                logger.exception('Unexpected error:')
                start_response('500 Internal Server Error', headerlist)
                content = ['Internal Server Error']
            if environ['REQUEST_METHOD'] == 'HEAD':
                return []
            else:
                return [string.encode('utf-8') for string in content]
Exemplo n.º 45
0
def test_no_auth_header(static_auth):
    request = mock.Mock()
    request.headers = Headers([])
    assert not static_auth.is_authorized(request)
Exemplo n.º 46
0
 def update(self, url=None, headers=None):
     if url:
         self.load(url)
     if headers:
         self.headers = Headers(headers)
Exemplo n.º 47
0
class BaseResponse:
    """Base class for Response."""
    default_status = 200
    default_content_type = 'text/plain;'

    def __init__(self, body=None, status=None, headers=None):
        self._body = body if body else [b'']
        self._status_code = status or self.default_status
        self.headers = Headers()
        self._cookies = SimpleCookie()

        if headers:
            for name, value in headers.items():
                self.headers.add_header(name, value)

    @property
    def body(self):
        return self._body

    @property
    def status_code(self):
        """ The HTTP status code as an integer (e.g. 404)."""
        return self._status_code

    @property
    def status(self):
        """ The HTTP status line as a string (e.g. ``404 Not Found``)."""
        status = _HTTP_STATUS_LINES.get(self._status_code)
        return str(status or ('{} Unknown'.format(self._status_code)))

    @status.setter
    def status(self, status_code):
        if not 100 <= status_code <= 999:
            raise ValueError('Status code out of range.')
        self._status_code = status_code

    @property
    def headerlist(self):
        """ WSGI conform list of (header, value) tuples. """
        if 'Content-Type' not in self.headers:
            self.headers.add_header('Content-Type', self.default_content_type)
        if self._cookies:
            for c in self._cookies.values():
                self.headers.add_header('Set-Cookie', c.OutputString())
        return self.headers.items()

    def set_cookie(self,
                   key,
                   value,
                   expires=None,
                   max_age=None,
                   path='/',
                   secret=None,
                   digestmod=hashlib.sha256):
        from kobin.app import current_config
        if secret is None:
            secret = current_config('SECRET_KEY')
        if secret:
            if isinstance(secret, str):
                secret = secret.encode('utf-8')
            encoded = base64.b64encode(
                pickle.dumps((key, value), pickle.HIGHEST_PROTOCOL))
            sig = base64.b64encode(
                hmac.new(secret, encoded, digestmod=digestmod).digest())
            value_bytes = b'!' + sig + b'?' + encoded
            value = value_bytes.decode('utf-8')

        self._cookies[key] = value
        if len(key) + len(value) > 3800:
            raise ValueError('Content does not fit into a cookie.')

        if max_age is not None:
            if isinstance(max_age, int):
                max_age_value = max_age
            else:
                max_age_value = max_age.seconds + max_age.days * 24 * 3600
            self._cookies[key]['max-age'] = max_age_value
        if expires is not None:
            if isinstance(expires, int):
                expires_value = expires
            else:
                expires_value = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
                                              expires.timetuple())
            self._cookies[key]['expires'] = expires_value
        if path:
            self._cookies[key]['path'] = path

    def delete_cookie(self, key, **kwargs):
        kwargs['max_age'] = -1
        kwargs['expires'] = 0
        self.set_cookie(key, '', **kwargs)
Exemplo n.º 48
0
class Response:
    default_status = 200
    default_content_type = 'text/html; charset=UTF-8'

    def __init__(self, body: str='', status: int=None, headers: Dict=None,
                 **more_headers) -> None:
        self.headers = Headers()
        self.body = body
        self._status_code = status or self.default_status
        self._cookies = SimpleCookie()  # type: ignore

        if headers:
            for name, value in headers.items():
                self.headers.add_header(name, value)
        if more_headers:
            for name, value in more_headers.items():
                self.headers.add_header(name, value)

    @property
    def status_code(self):
        """ The HTTP status code as an integer (e.g. 404)."""
        return self._status_code

    @property
    def status(self):
        """ The HTTP status line as a string (e.g. ``404 Not Found``)."""
        if not 100 <= self._status_code <= 999:
            raise ValueError('Status code out of range.')
        status = _HTTP_STATUS_LINES.get(self._status_code)
        return str(status or ('{} Unknown'.format(self._status_code)))

    @status.setter
    def status(self, status_code: int):
        if not 100 <= status_code <= 999:
            raise ValueError('Status code out of range.')
        self._status_code = status_code

    @property
    def headerlist(self) -> List[Tuple[str, str]]:
        """ WSGI conform list of (header, value) tuples. """
        out = []  # type: List[Tuple[str, str]]
        if 'Content-Type' not in self.headers:
            self.headers.add_header('Content-Type', self.default_content_type)
        out += [(key, value)
                for key in self.headers.keys()
                for value in self.headers.get_all(key)]
        if self._cookies:
            for c in self._cookies.values():
                out.append(('Set-Cookie', c.OutputString()))
        return [(k, v.encode('utf8').decode('latin1')) for (k, v) in out]

    def set_cookie(self, key: str, value: Any, expires: str=None, path: str=None, **options: Dict[str, Any]) -> None:
        from datetime import timedelta, datetime, date
        import time
        self._cookies[key] = value
        if expires:
            self._cookies[key]['expires'] = expires
        if path:
            self._cookies[key]['path'] = path

        for k, v in options.items():
            if k == 'max_age':
                if isinstance(v, timedelta):
                    v = v.seconds + v.days * 24 * 3600  # type: ignore
            if k == 'expires':
                if isinstance(v, (date, datetime)):
                    v = v.timetuple()  # type: ignore
                elif isinstance(v, (int, float)):
                    v = v.gmtime(value)  # type: ignore
                v = time.strftime("%a, %d %b %Y %H:%M:%S GMT", v)  # type: ignore
            self._cookies[key][k.replace('_', '-')] = v  # type: ignore

    def delete_cookie(self, key, **kwargs) -> None:
        kwargs['max_age'] = -1
        kwargs['expires'] = 0
        self.set_cookie(key, '', **kwargs)

    def apply(self, other):
        self.status = other._status_code
        self._cookies = other._cookies
        self.headers = other.headers
        self.body = other.body
Exemplo n.º 49
0
 def __init__(self: 'Response', start_response, request: Request):
     self.request = request
     self.id = request.id
     self._start_response = start_response
     self.headers = Headers()
     self.status_code = 200
Exemplo n.º 50
0
    def run_app(self, conn):
        self.status = "200 OK"
        self.size = 0
        self.expires = None
        self.etag = None
        self.content_type = 'text/plain'
        self.content_length = None

        if __debug__:
            self.err_log.debug('Getting sock_file')

        # Build our file-like object
        sock_file = conn.makefile('rb', BUF_SIZE)
        request = self.read_request_line(sock_file)
        if request['method'].upper() not in ('GET', ):
            self.status = "501 Not Implemented"

        try:
            # Get our file path
            headers = dict([(str(k.lower()), v)
                            for k, v in self.read_headers(sock_file).items()])
            rpath = request.get('path', '').lstrip('/')
            filepath = os.path.join(self.root, rpath)
            filepath = os.path.abspath(filepath)
            if __debug__:
                self.err_log.debug('Request for path: %s' % filepath)

            self.closeConnection = headers.get('connection',
                                               'close').lower() == 'close'
            self.headers = Headers([
                ('Date', formatdate(usegmt=True)),
                ('Server', HTTP_SERVER_SOFTWARE),
                ('Connection', headers.get('connection', 'close')),
            ])

            if not filepath.lower().startswith(self.root.lower()):
                # File must be within our root directory
                self.status = "400 Bad Request"
                self.closeConnection = True
            elif not os.path.exists(filepath):
                self.status = "404 File Not Found"
                self.closeConnection = True
            elif os.path.isdir(filepath):
                self.serve_dir(filepath, rpath)
            elif os.path.isfile(filepath):
                self.serve_file(filepath, headers)
            else:
                # It exists but it's not a file or a directory????
                # What is it then?
                self.status = "501 Not Implemented"
                self.closeConnection = True

            h = self.headers
            statcode, statstr = self.status.split(' ', 1)
            statcode = int(statcode)
            if statcode >= 400:
                h.add_header('Content-Type', self.content_type)
                self.data = [statstr]

            # Build our output headers
            header_data = HEADER_RESPONSE % (self.status, str(h))

            # Send the headers
            if __debug__:
                self.err_log.debug('Sending Headers: %s' % repr(header_data))
            self.conn.sendall(b(header_data))

            for data in self.data:
                self.conn.sendall(b(data))

            if hasattr(self.data, 'close'):
                self.data.close()

        finally:
            if __debug__:
                self.err_log.debug('Finally closing sock_file')
            sock_file.close()
Exemplo n.º 51
0
class Handler(object):

    badidre = re.compile(r"[<>\s]")

    def __init__(self, service, siptype, wsgienv, start_resp, auth=None):
        self._svc = service
        self._env = wsgienv
        self._start = start_resp
        self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
        self._hdr = Headers([])
        self._code = 0
        self._msg = "unknown status"
        self._auth = auth

    def send_error(self, code, message):
        stat = "{0} {1}".format(str(code), message)
        self._start(stat, [], sys.exc_info())

    def add_header(self, name, value):
        # Caution: HTTP does not support Unicode characters (see
        # https://www.python.org/dev/peps/pep-0333/#unicode-issues);
        # thus, this will raise a UnicodeEncodeError if the input strings
        # include Unicode (char code > 255).
        e = "ISO-8859-1"
        self._hdr.add_header(name.encode(e), value.encode(e))

    def set_response(self, code, message):
        self._code = code
        self._msg = message

    def end_headers(self):
        stat = "{0} {1}".format(str(self._code), self._msg)
        self._start(stat, self._hdr.items())

    def handle(self):
        meth_handler = 'do_' + self._meth

        path = self._env.get('PATH_INFO', '/').strip('/')
        if not self.authorize():
            return self.send_unauthorized()

        if hasattr(self, meth_handler):
            out = getattr(self, meth_handler)(path)
            if isinstance(out, list) and len(out) > 0:
                out.append('\n')
            return out
        else:
            return self.send_error(
                403, self._meth + " not supported on this resource")

    def authorize(self):
        if self._auth[0] == 'header':
            return self.authorize_via_headertoken()
        else:
            return self.authorize_via_queryparam()

    def authorize_via_queryparam(self):
        params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
        auths = params.get('auth', [])
        if self._auth[1]:
            # match the last value provided
            return len(auths) > 0 and self._auth[1] == auths[-1]
        if len(auths) > 0:
            log.warn(
                "Authorization key provided, but none has been configured")
        return len(auths) == 0

    def authorize_via_headertoken(self):
        authhdr = self._env.get('HTTP_AUTHORIZATION', "")
        parts = authhdr.split()
        if self._auth[1]:
            return len(parts) > 1 and parts[0] == "Bearer" and \
                self._auth[1] == parts[1]
        if authhdr:
            log.warn(
                "Authorization key provided, but none has been configured")
        return authhdr == ""

    def send_unauthorized(self):
        self.set_response(401, "Not authorized")
        if self._auth[0] == 'header':
            self.add_header('WWW-Authenticate', 'Bearer')
        self.end_headers()
        return []

    def do_GET(self, path):
        # return the status on request or a list of previous requests
        steps = path.split('/')
        if steps[0] == '':
            try:
                out = json.dumps(['midas'])
            except Exception, ex:
                log.exception("Internal error: " + str(ex))
                self.send_error(500, "Internal error")
                return ["[]"]

            self.set_response(200, "Supported SIP Types")
            self.add_header('Content-Type', 'application/json')
            self.end_headers()
            return [out]

        elif steps[0] == 'midas':
            if len(steps) > 2:
                path = '/'.join(steps[1:])
                self.send_error(400, "Unsupported SIP identifier: " + path)
                return []
            elif len(steps) > 1:
                if steps[1].startswith("_") or steps[1].startswith(".") or \
                   self.badidre.search(steps[1]):

                    self.send_error(400, "Unsupported SIP identifier: " + path)
                    return []

                return self.request_status(steps[1])

            else:
                return self.requests()
Exemplo n.º 52
0
    def run_app(self, conn):
        self.size = 0
        self.header_set = Headers([])
        self.headers_sent = False
        self.error = (None, None)
        self.chunked = False
        sections = None
        output = None

        if __debug__:
            self.err_log.debug('Getting sock_file')

        # Build our file-like object
        if PY3K:
            sock_file = conn.makefile(mode='rb', buffering=BUF_SIZE)
        else:
            sock_file = conn.makefile(BUF_SIZE)

        try:
            # Read the headers and build our WSGI environment
            self.environ = environ = self.build_environ(sock_file, conn)

            # Handle 100 Continue
            if environ.get('HTTP_EXPECT', '') == '100-continue':
                res = environ['SERVER_PROTOCOL'] + ' 100 Continue\r\n\r\n'
                conn.sendall(b(res))

            # Send it to our WSGI application
            output = self.app(environ, self.start_response)

            if not hasattr(output, '__len__') and not hasattr(
                    output, '__iter__'):
                self.error = ('500 Internal Server Error',
                              'WSGI applications must return a list or '
                              'generator type.')

            if hasattr(output, '__len__'):
                sections = len(output)

            for data in output:
                # Don't send headers until body appears
                if data:
                    self.write(data, sections)

            if not self.headers_sent:
                # Send headers if the body was empty
                self.send_headers('', sections)

            if self.chunked and self.request_method != 'HEAD':
                # If chunked, send our final chunk length
                self.conn.sendall(b('0\r\n\r\n'))

        # Don't capture exceptions here.  The Worker class handles
        # them appropriately.
        finally:
            if __debug__:
                self.err_log.debug('Finally closing output and sock_file')

            if hasattr(output, 'close'):
                output.close()

            sock_file.close()
Exemplo n.º 53
0
 def __init__(self, status, content_type):
     self.__status = status
     self.__ct = content_type
     self.__header = Headers()
Exemplo n.º 54
0
def test_wrong_token(static_auth):
    request = mock.Mock()
    request.headers = Headers([("authorization", "Bearer wrong")])
    assert not static_auth.is_authorized(request)
Exemplo n.º 55
0
class Route():

    def __init__(self, path, method, callback, status=200, content_type=None):
        self.path = path
        self.method = method
        self.callback = callback

        self.__status = status

        if content_type is None: content_type = 'text/html; charset=UTF-8'
        self.__ct = content_type

        self.__header = Headers()

    @property
    def status_code(self):
        return '{} {}'.format(self.__status, responses[self.__status])

    @property
    def headers(self):
        self.__header.add_header('Strict-Transport-Security', 'max-age=31536000; includeSubDomains')
        self.__header.add_header('Content-Security-Policy', "default-src 'self'")
        self.__header.add_header('X-Content-Type-Options', 'nosniff')
        self.__header.add_header('X-Frame-Options', 'SAMEORIGIN')
        self.__header.add_header('X-XSS-Protection', '1; mode=block')
        self.__header.add_header('Content-type', self.__ct)
        return self.__header.items()
Exemplo n.º 56
0
    def getTileResponse(self, coord, extension, ignore_cached=False):
        """ Get status code, headers, and a tile binary for a given request layer tile.

            Arguments:
            - coord: one ModestMaps.Core.Coordinate corresponding to a single tile.
            - extension: filename extension to choose response type, e.g. "png" or "jpg".
            - ignore_cached: always re-render the tile, whether it's in the cache or not.

            This is the main entry point, after site configuration has been loaded
            and individual tiles need to be rendered.
        """
        start_time = time()

        mimetype, format = self.getTypeByExtension(extension)

        # default response values
        status_code = 200
        headers = Headers([('Content-Type', mimetype)])
        body = None

        cache = self.config.cache

        if not ignore_cached:
            # Start by checking for a tile in the cache.
            try:
                body = cache.read(self, coord, format)
            except TheTileLeftANote as e:
                headers = e.headers
                status_code = e.status_code
                body = e.content

                if e.emit_content_type:
                    headers.setdefault('Content-Type', mimetype)

            tile_from = 'cache'

        else:
            # Then look in the bag of recent tiles.
            body = _getRecentTile(self, coord, format)
            tile_from = 'recent tiles'

        # If no tile was found, dig deeper
        if body is None:
            try:
                lockCoord = None

                if self.write_cache:
                    # this is the coordinate that actually gets locked.
                    lockCoord = self.metatile.firstCoord(coord)

                    # We may need to write a new tile, so acquire a lock.
                    cache.lock(self, lockCoord, format)

                if not ignore_cached:
                    # There's a chance that some other process has
                    # written the tile while the lock was being acquired.
                    body = cache.read(self, coord, format)
                    tile_from = 'cache after all'

                if body is None:
                    # No one else wrote the tile, do it here.
                    buff = BytesIO()

                    try:
                        tile = self.render(coord, format)
                        save = True
                    except NoTileLeftBehind as e:
                        tile = e.tile
                        save = False
                        status_code = 404

                    if not self.write_cache:
                        save = False

                    if format.lower() == 'jpeg':
                        save_kwargs = self.jpeg_options
                    elif format.lower() == 'png':
                        save_kwargs = self.png_options
                    else:
                        save_kwargs = {}

                    tile.save(buff, format, **save_kwargs)
                    body = buff.getvalue()

                    if save:
                        cache.save(body, self, coord, format)

                    tile_from = 'layer.render()'

            except TheTileLeftANote as e:
                headers = e.headers
                status_code = e.status_code
                body = e.content

                if e.emit_content_type:
                    headers.setdefault('Content-Type', mimetype)

            finally:
                if lockCoord:
                    # Always clean up a lock when it's no longer being used.
                    cache.unlock(self, lockCoord, format)

        _addRecentTile(self, coord, format, body)
        logging.info(
            'TileStache.Core.Layer.getTileResponse() %s/%d/%d/%d.%s via %s in %.3f',
            self.name(), coord.zoom, coord.column, coord.row, extension,
            tile_from,
            time() - start_time)

        return status_code, headers, body
Exemplo n.º 57
0
class Handler(object):

    def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None, postexec=None):
        self._env = wsgienv
        self._start = start_resp
        self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
        self._hdr = Headers([])
        self._code = 0
        self._msg = "unknown status"
        self._auth = auth
        self._archdir = archdir
        self._postexec = postexec

        self._loaders = loaders

    def send_error(self, code, message):
        status = "{0} {1}".format(str(code), message)
        self._start(status, [], sys.exc_info())
        return []

    def add_header(self, name, value):
        self._hdr.add_header(name, value)

    def set_response(self, code, message):
        self._code = code
        self._msg = message

    def end_headers(self):
        status = "{0} {1}".format(str(self._code), self._msg)
        self._start(status, list(self._hdr.items()))

    def handle(self):
        meth_handler = 'do_'+self._meth

        path = self._env.get('PATH_INFO', '/')[1:]
        if not self.authorize():
            return self.send_unauthorized()

        if hasattr(self, meth_handler):
            return getattr(self, meth_handler)(path)
        else:
            return self.send_error(403, self._meth +
                                   " not supported on this resource")

    def authorize(self):
        if self._auth[0] == 'header':
            return self.authorize_via_headertoken()
        else:
            return self.authorize_via_queryparam()

    def authorize_via_queryparam(self):
        params = parse_qs(self._env.get('QUERY_STRING', ''))
        auths = params.get('auth',[])
        if self._auth[1]:
            # match the last value provided
            return len(auths) > 0 and self._auth[1] == auths[-1]  
        if len(auths) > 0:
            log.warning("Authorization key provided, but none has been configured")
        return len(auths) == 0

    def authorize_via_headertoken(self):
        authhdr = self._env.get('HTTP_AUTHORIZATION', "")
        log.debug("Request HTTP_AUTHORIZATION: %s", authhdr)
        parts = authhdr.split()
        if self._auth[1]:
            return len(parts) > 1 and parts[0] == "Bearer" and \
                self._auth[1] == parts[1]
        if authhdr:
            log.warning("Authorization key provided, but none has been configured")
        return authhdr == ""

    def send_unauthorized(self):
        self.set_response(401, "Not authorized")
        if self._auth[0] == 'header':
            self.add_header('WWW-Authenticate', 'Bearer')
        self.end_headers()
        return []

    def do_GET(self, path):
        path = path.strip('/')
        if not path:
            try:
                out = json.dumps(list(self._loaders.keys())) + '\n'
                out = out.encode()
            except Exception as ex:
                log.exception("Internal error: "+str(ex))
                return self.send_error(500, "Internal error")

            self.set_response(200, "Supported Record Types")
            self.add_header('Content-Type', 'application/json')
            self.add_header('Content-Length', str(len(out)))
            self.end_headers()
            return [out]
        elif path in self._loaders:
            self.set_response(200, "Service is ready")
            self.add_header('Content-Type', 'application/json')
            self.end_headers()
            return [b"Service ready\n"]
        else:
            return self.send_error(404, "resource does not exist")
            
    def do_POST(self, path):
        path = path.strip('/')
        steps = path.split('/')
        if len(steps) == 0:
            return self.send_error(405, "POST not supported on this resource")
        elif len(steps) == 1:
            if steps[0] == 'nerdm':
                return self.ingest_nerdm_record()
            else:
                return self.send_error(403, "new records are not allowed for " +
                                       "submission to this resource")
        else:
            return self.send_error(404, "resource does not exist")

    def nerdm_archive_cache(self, rec):
        """
        cache a NERDm record into a local disk archive.  The cache is for 
        records that have been accepted but not ingested.  
        """
        try:
            arkid = re.sub(r'/.*$', '', re.sub(r'ark:/\d+/', '', rec['@id']))
            ver = rec.get('version', '1.0.0').replace('.', '_')
            recid = "%s-v%s" % (os.path.basename(arkid), ver)
            outfile = os.path.join(self._archdir, '_cache', recid+".json")
            with open(outfile, 'w') as fd:
                json.dump(rec, fd, indent=2)

            return recid
        
        except KeyError as ex:
            # this shouldn't happen if the record was already validated
            raise RecordIngestError("submitted record is missing the @id "+
                                    "property")
        except ValueError as ex:
            # this shouldn't happen if the record was already validated
            raise RecordIngestError("submitted record is apparently invalid; "+
                                    "unable to submit")
        except OSError as ex:
            raise RuntimeError("Failed to cache record ({0}): {1}"
                               .format(arkid, str(ex)))

    def nerdm_archive_commit(self, recid):
        """
        commit a previously cached record to the local disk archive.  This
        method is called after the record has been successfully ingested to
        the RMM's database.
        """
        outfile = os.path.join(self._archdir, '_cache', recid+".json")
        if not os.path.exists(outfile):
            raise RuntimeError("record to commit ({0}) not found in cache: {1}"
                               .format(recid, outfile))
        try:
            os.rename(outfile,
                      os.path.join(self._archdir, os.path.basename(outfile)))
        except OSError as ex:
            raise RuntimeError("Failed to archvie record ({0}): {1}"
                               .format(recid, str(ex)))
        

    def ingest_nerdm_record(self):
        """
        Accept a NERDm record for ingest into the RMM
        """
        loader = self._loaders['nerdm']

        try:
            clen = int(self._env['CONTENT_LENGTH'])
        except KeyError as ex:
            log.exception("Content-Length not provided for input record")
            return self.send_error(411, "Content-Length is required")
        except ValueError as ex:
            log.exception("Failed to parse input JSON record: "+str(e))
            return self.send_error(400, "Content-Length is not an integer")

        try:
            bodyin = self._env['wsgi.input']
            doc = bodyin.read(clen)
            rec = json.loads(doc)
        except Exception as ex:
            log.exception("Failed to parse input JSON record: "+str(ex))
            log.warning("Input document starts...\n{0}...\n...{1} ({2}/{3} chars)"
                        .format(doc[:75], doc[-20:], len(doc), clen))
            return self.send_error(400,
                                   "Failed to load input record (bad format?): "+
                                   str(ex))

        try:
            recid = self.nerdm_archive_cache(rec)
            
            res = loader.load(rec, validate=True)
            if res.failure_count > 0:
                res = res.failures()[0]
                logmsg = "Failed to load record with "+str(res.key)
                for e in res.errs:
                    logmsg += "\n  "+str(e)
                log.error(logmsg)
                self.set_response(400, "Input record is not valid")
                self.add_header('Content-Type', 'application/json')
                self.end_headers()
                out = json.dumps([str(e) for e in res.errs]) + '\n'
                return [ out.encode() ]

        except RecordIngestError as ex:
            log.exception("Failed to load posted record: "+str(ex))
            self.set_response(400, "Input record is not valid (missing @id)")
            self.add_header('Content-Type', 'application/json')
            self.end_headers()
            out = json.dumps([ "Record is missing @id property" ]) + '\n'
            return [ out.encode() ]

        except Exception as ex:
            log.exception("Loading error: "+str(ex))
            return self.send_error(500, "Load failure due to internal error")

        try:
            self.nerdm_archive_commit(recid)
        except Exception as ex:
            log.exception("Commit error: "+str(ex))

        if self._postexec:
            # run post-commit script
            try:
                self.nerdm_post_commit(recid)
            except Exception as ex:
                log.exception("Post-commit error: "+str(ex))

        log.info("Accepted record %s with @id=%s",
                 rec.get('ediid','?'), rec.get('@id','?'))
        self.set_response(200, "Record accepted")
        self.end_headers()
        return []

    def nerdm_post_commit(self, recid):
        """
        run an external executable for further processing after the record is commited to 
        the database (e.g. update an external index)
        """
        cmd = _mkpostcomm(self._postexec, recid)

        try:
            log.debug("Executing post-commit script:\n  %s", " ".join(cmd))
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (out, err) = p.communicate()
            if p.returncode != 0:
                log.error("Error occurred while running post-commit script:\n"+(err or out))
        except OSError as ex:
            log.error("Failed to execute post-commit script:\n  %s\n%s", " ".join(cmd), str(ex))
        except Exception as ex:
            log.error("Unexpected failure executing post-commit script:\n  %s\n%s", " ".join(cmd), str(ex))
Exemplo n.º 58
0
class Url(Packer, object):
    def __init__(self,
                 id,
                 url,
                 cookie='',
                 headers=HEADERS_CHROME,
                 host=None,
                 port=None,
                 path=None,
                 protocol=None,
                 proxy=None,
                 max_thread=-1,
                 range_format='Range: bytes=%d-%d'):

        self.id = id

        self.url = url

        self.host = host if host is not None else getattr(self, 'host', None)
        self.port = port if port is not None else getattr(self, 'port', None)

        self.path = path if path is not None else getattr(self, 'path', None)
        self.protocol = protocol if protocol is not None else getattr(
            self, 'protocol', None)

        self.cookie = cookie

        if isinstance(headers, Headers):
            self.headers = headers
        elif isinstance(headers, dict):
            self.headers = Headers(list(headers.items()))
        else:
            raise ValueError('headers must be an instance of dict or Headers')

        self.etag = None

        self.proxy = proxy
        self.target = Target()

        self.max_thread = max_thread

        self.range_format = range_format

    def __eq__(self, other):
        if isinstance(other, Url):
            return self.url == other.url and \
                self.cookie == other.cookie and \
                self.proxy == other.proxy and \
                self.range_format == other.range_format
        else:
            object.__eq__(self, other)

    def config(self):
        pass

    def getContentSize(self):
        if self.target.code == 200 and int(
                self.target.headers.get('Content-Length', -1)) != -1:
            return int(self.target.headers.get('Content-Length'))
        elif self.target.code == 206 and self.target.headers.get(
                'Content-Range'):
            return int(self.target.headers.get('Content-Range').split('/')[-1])
        else:
            return -1

    def getFileName(self):

        ctd = self.target.headers.get('Content-Disposition')
        if ctd is not None:
            filename = re.findall(r'filename="(.*?)"', ctd)
            if filename:
                return filename[0]

        filename = self.path.split('?')[0].split('/')[-1]

        if filename != '':
            if '.' not in filename or filename.split('.')[-1] == '':

                extension = _content_type(
                    self.target.headers.get('Content-Type'))
                filename = filename + extension

        else:
            filename = None

        return filename

    def reload(self):
        self.target.load(self.url)

    def __setattr__(self, key, value):
        object.__setattr__(self, key, value)
        if key == 'url':
            self.protocol, s1 = splittype(self.url)
            if s1:
                s2, self.path = splithost(s1)
                if s2:
                    self.host, port = splitport(s2)
                    self.port = int(port) if port is not None else None

            if not getattr(self, 'port', None):
                if self.protocol == 'http':
                    self.port = 80
                elif self.protocol == 'https':
                    self.port = 443

    def activate(self):
        res, cookie_dict = self.__request__()
        # if res.getcode() == 200 or res.getcode() == 206:
        headers_items = ()
        if sys.version_info < (3, 0):
            headers_items = res.info().items()

        if sys.version_info >= (3, 0):
            headers_items = res.getheaders()
        self.target.update(res.geturl(), headers_items, res.getcode())
        # else:
        #     raise Exception('UrlNoRespond or UrlError')

    def __request__(self):

        Cookiejar = CookieJar()
        opener = build_opener(HTTPCookieProcessor(Cookiejar))
        _header = dict(self.headers.items())
        if self.cookie:
            _header.update({'Cookie': self.cookie})
        req = Request(self.url, headers=_header, origin_req_host=self.host)
        error_counter = 0
        while error_counter < 3:
            try:
                res = opener.open(req)
                break
            except Exception as e:
                # traceback.print_exc()
                error_counter += 1
            time.sleep(0.5)
        else:
            raise Exception('UrlNotRespond')

        return res, Cookiejar._cookies

    def getHeader(self, name, default=None):
        return self.headers.get(name, default)

    def __packet_params__(self):
        return [
            'id', 'url', 'host', 'port', 'protocal', 'cookie', 'etag', 'proxy',
            'max_thread', 'range_format', 'headers'
        ]
Exemplo n.º 59
0
class MultipartPart(object):

    def __init__(self, buffer_size=2 ** 16, memfile_limit=2 ** 18, charset='latin1'):
        self.headerlist = []
        self.headers = None
        self.file = False
        self.size = 0
        self._buf = tob('')
        self.disposition, self.name, self.filename = None, None, None
        self.content_type, self.charset = None, charset
        self.memfile_limit = memfile_limit
        self.buffer_size = buffer_size

    def feed(self, line, nl=''):
        if self.file:
            return self.write_body(line, nl)
        return self.write_header(line, nl)

    def write_header(self, line, nl):
        line = line.decode(self.charset or 'latin1')
        if not nl:
            raise MultipartError('Unexpected end of line in header.')
        if not line.strip():  # blank line -> end of header segment
            self.finish_header()
        elif line[0] in ' \t' and self.headerlist:
            name, value = self.headerlist.pop()
            self.headerlist.append((name, value + line.strip()))
        else:
            if ':' not in line:
                raise MultipartError("Syntax error in header: No colon.")
            name, value = line.split(':', 1)
            self.headerlist.append((name.strip(), value.strip()))

    def write_body(self, line, nl):
        if not line and not nl:
            return  # This does not even flush the buffer
        self.size += len(line) + len(self._buf)
        self.file.write(self._buf + line)
        self._buf = nl
        if self.content_length > 0 and self.size > self.content_length:
            raise MultipartError('Size of body exceeds Content-Length header.')
        if self.size > self.memfile_limit and isinstance(self.file, BytesIO):
            # TODO: What about non-file uploads that exceed the memfile_limit?
            self.file, old = TemporaryFile(mode='w+b'), self.file
            old.seek(0)
            copy_file(old, self.file, self.size, self.buffer_size)

    def finish_header(self):
        self.file = BytesIO()
        self.headers = Headers(self.headerlist)
        cdis = self.headers.get('Content-Disposition', '')
        ctype = self.headers.get('Content-Type', '')
        self.headers.get('Content-Length', '-1')
        if not cdis:
            raise MultipartError('Content-Disposition header is missing.')
        self.disposition, self.options = parse_options_header(cdis)
        self.name = self.options.get('name')
        self.filename = self.options.get('filename')
        self.content_type, options = parse_options_header(ctype)
        self.charset = options.get('charset') or self.charset
        self.content_length = int(self.headers.get('Content-Length', '-1'))

    def is_buffered(self):
        ''' Return true if the data is fully buffered in memory.'''
        return isinstance(self.file, BytesIO)

    @property
    def value(self):
        ''' Data decoded with the specified charset '''
        pos = self.file.tell()
        self.file.seek(0)
        val = self.file.read()
        self.file.seek(pos)
        return val.decode(self.charset)

    def save_as(self, path):
        fp = open(path, 'wb')
        pos = self.file.tell()
        try:
            self.file.seek(0)
            size = copy_file(self.file, fp)
        finally:
            self.file.seek(pos)
        return size
Exemplo n.º 60
0
 def _handle_http_response_start(self, message: Dict[str, Any]):
     self._headers = Headers(
         [(k.decode(), v.decode())
          for k, v in message["headers"]])
     self._status_code = message["status"]