def handle_application_error(self, environ, start_response): status = "500 Internal Server Error" headers = Headers([]) # Package the exception info as into a special header and # send it to the client type, exc, tb = sys.exc_info() tbfile = StringIO() traceback.print_exc(file=tbfile) headers['Content-Type'] = 'text/plain; charset=utf-8' LOG.debug("Packing traceback context into debug header: %s", self.debug_header) debug_header = self.pack_header(Traceback(tb)) LOG.debug("Debug header (%d bytes): %s", len(debug_header), debug_header) headers[self.debug_header] = debug_header app_uri = application_uri(environ) headers["Location"] = app_uri[:-1] + self.debug_uri start_response(status, headers.items()) return [tbfile.getvalue().encode('utf-8')]
def patched_start_response(status, headers, exc_info=None): # if self._should_handle(headers) wsgi_headers = Headers(headers) # If we're debugging, or the response already has an expires # header, just skip this. if not self.debug and "Expires" not in wsgi_headers: mime = wsgi_headers.get("Content-Type", "*").split(";")[0] # If the mime type is explicitly called out, use the expire # delay specified. if mime in self.expire_seconds: expire_time = self.make_expire_time_for(mime) # If there's a catch-all wildcard delay, use that. elif "*" in self.expire_seconds: expire_time = self.make_expire_time_for("*") # Otherwise, don't set the header. else: expire_time = None if expire_time is not None: log.debug("Adding expires header value: " + expire_time) headers.append(("Expires", expire_time)) return start_response(status, headers, exc_info)
def thread_app(env, resp): path = env['PATH_INFO'] # utils.log('thread_app', path) m = thread_re.match(path) board, datkey = m.group(1), m.group(2) key = keylib.get_filekey(datkey) data = cache.Cache(key) data.load() if check_get_cache(env): if not data.exists() or len(data) == 0: # when first access, load data from network data.search() elif _count_is_update(key): # update thread # limit `data.search` calling. it's slow! threading.Thread(target=data.search, daemon=True).start() if not data.exists(): resp('404 Not Found', [('Content-Type', 'text/plain; charset=Shift_JIS')]) return [b'404 Not Found'] thread = dat.make_dat(data, env, board) headers = Headers([('Content-Type', 'text/plain; charset=Shift_JIS')]) last_m = eutils.formatdate(data.stamp) headers['Last-Modified'] = last_m resp("200 OK", headers.items()) return (c.encode('cp932', 'replace') for c in thread)
def patched_start_response(status, headers, exc_info=None): # if self._should_handle(headers) wsgi_headers = Headers(headers) # If we're debugging, or the response already has an expires # header, just skip this. log.debug('Skipping expired headers' if self.debug else 'Calculating expires headers') if not self.debug and 'Expires' not in wsgi_headers: mime = wsgi_headers.get('Content-Type', '*').split(';')[0] log.debug('See mime type ' + mime) # If the mime type is explicitly called out, use the expire # delay specified. if mime in self.expire_seconds: log.debug('Matched mimetype exactly.') expire_time = self.make_expire_time_for(mime) # If there's a catch-all wildcard delay, use that. elif '*' in self.expire_seconds: log.debug('Matched mimetype with universal.') expire_time = self.make_expire_time_for('*') # Otherwise, don't set the header. else: log.debug('No mimetype match.') expire_time = None if expire_time is not None: log.debug('Adding expires header value: ' + expire_time) headers.append(('Expires', expire_time)) log.debug('-'*60) return start_response(status, headers, exc_info)
def board_app(env, resp): path = env['PATH_INFO'] m = board_re.match(path) board = m.group(1) message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja')) headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')]) resp("200 OK", headers.items()) board = utils.sanitize(utils.get_board(path)) if board: fmt = '{logo} - {board} - {desc}' else: fmt = '{logo} - {desc}' text = fmt.format(logo=message['logo'], desc=message['description'], board=board) html = ''' <!DOCTYPE html> <html><head> <meta http-equiv="content-type" content="text/html; charset=Shift_JIS"> <title>{text}</title> <meta name="description" content="{text}"> </head><body> <h1>{text}</h1> </body></html> '''.format(text=text) return [html.encode('cp932', 'replace')]
def testMappingInterface(self): test = [("x", "y")] self.assertEqual(len(Headers([])), 0) self.assertEqual(len(Headers(test[:])), 1) self.assertEqual(Headers(test[:]).keys(), ["x"]) self.assertEqual(Headers(test[:]).values(), ["y"]) self.assertEqual(Headers(test[:]).items(), test) self.assertIsNot(Headers(test).items(), test) # must be copy! h = Headers([]) del h["foo"] # should not raise an error h["Foo"] = "bar" for m in h.has_key, h.__contains__, h.get, h.get_all, h.__getitem__: self.assertTrue(m("foo")) self.assertTrue(m("Foo")) self.assertTrue(m("FOO")) self.assertFalse(m("bar")) self.assertEqual(h["foo"], "bar") h["foo"] = "baz" self.assertEqual(h["FOO"], "baz") self.assertEqual(h.get_all("foo"), ["baz"]) self.assertEqual(h.get("foo", "whee"), "baz") self.assertEqual(h.get("zoo", "whee"), "whee") self.assertEqual(h.setdefault("foo", "whee"), "baz") self.assertEqual(h.setdefault("zoo", "whee"), "whee") self.assertEqual(h["foo"], "baz") self.assertEqual(h["zoo"], "whee")
def testMappingInterface(self): test = [('x','y')] self.assertEqual(len(Headers([])),0) self.assertEqual(len(Headers(test[:])),1) self.assertEqual(Headers(test[:]).keys(), ['x']) self.assertEqual(Headers(test[:]).values(), ['y']) self.assertEqual(Headers(test[:]).items(), test) self.assertFalse(Headers(test).items() is test) # must be copy! h=Headers([]) del h['foo'] # should not raise an error h['Foo'] = 'bar' for m in h.__contains__, h.get, h.get_all, h.__getitem__: self.assertTrue(m('foo')) self.assertTrue(m('Foo')) self.assertTrue(m('FOO')) self.assertFalse(m('bar')) self.assertEqual(h['foo'],'bar') h['foo'] = 'baz' self.assertEqual(h['FOO'],'baz') self.assertEqual(h.get_all('foo'),['baz']) self.assertEqual(h.get("foo","whee"), "baz") self.assertEqual(h.get("zoo","whee"), "whee") self.assertEqual(h.setdefault("foo","whee"), "baz") self.assertEqual(h.setdefault("zoo","whee"), "whee") self.assertEqual(h["foo"],"baz") self.assertEqual(h["zoo"],"whee")
def parse_http_headers(environ): h = Headers([]) for k, v in environ.items(): if k.startswith('HTTP_'): name = k[5:] h.add_header(name, v) return h
def getTileResponse(self, coord, extension, ignore_cached=False): """ Get status code, headers, and a tile binary for a given request layer tile. Arguments: - coord: one ModestMaps.Core.Coordinate corresponding to a single tile. - extension: filename extension to choose response type, e.g. "png" or "jpg". - ignore_cached: always re-render the tile, whether it's in the cache or not. This is the main entry point, after site configuration has been loaded and individual tiles need to be rendered. """ start_time = time() mimetype, format = self.getTypeByExtension(extension) # default response values status_code = 200 headers = Headers([('Content-Type', mimetype)]) body = None cache = self.config.cache if not ignore_cached: # Start by checking for a tile in the cache. try: body = cache.read(self, coord, format) except TheTileLeftANote, e: headers = e.headers status_code = e.status_code body = e.content if e.emit_content_type: headers.setdefault('Content-Type', mimetype) tile_from = 'cache'
class Response(threading.local): """ Represents a single response using thread-local namespace. """ def bind(self, app): """ Clears old data and creates a brand new Response object """ self._COOKIES = None self.status = 200 self.header_list = [] self.header = HeaderWrapper(self.header_list) self.charset = 'UTF-8' self.content_type = 'text/html; charset=UTF-8' self.error = None self.app = app def add_header(self, key, value): self.header.add_header(key.title(), str(value)) def wsgiheaders(self): ''' Returns a wsgi conform list of header/value pairs ''' for c in self.COOKIES.values(): self.add_header('Set-Cookie', c.OutputString()) return self.header_list @property def COOKIES(self): if not self._COOKIES: self._COOKIES = SimpleCookie() return self._COOKIES def set_cookie(self, key, value, **kargs): """ Sets a Cookie. Optional settings: expires, path, comment, domain, max-age, secure, version, httponly """ if not isinstance(value, basestring): sec = self.app.config['securecookie.key'] value = cookie_encode(value, sec) self.COOKIES[key] = value for k, v in kargs.iteritems(): self.COOKIES[key][k] = v def get_content_type(self): """ Get the current 'Content-Type' header. """ return self.header['Content-Type'] def set_content_type(self, value): if 'charset=' in value: self.charset = value.split('charset=')[-1].split(';')[0].strip() self.header['Content-Type'] = value content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__)
def get_alternatives(base_headers, files): # Sort by size so that the smallest compressed alternative matches first alternatives = [] files_by_size = sorted(files.items(), key=lambda i: i[1].stat.st_size) for encoding, file_entry in files_by_size: headers = Headers(base_headers.items()) headers['Content-Length'] = str(file_entry.stat.st_size) if encoding: headers['Content-Encoding'] = encoding encoding_re = re.compile(r'\b%s\b' % encoding) else: encoding_re = re.compile('') alternatives.append((encoding_re, file_entry.path, headers.items())) return alternatives
def signal_land_or_sea(self, body, layer, coord, format): if body: md5sum = self.md5sum(body) second_md5sum = self.md5sum(Disk.read(self, self.second, coord, format)) headers = Headers([('Access-Control-Expose-Headers', 'X-Land-Or-Sea')]) headers.setdefault('X-Land-Or-Sea', '0') if second_md5sum and md5sum == second_md5sum: if md5sum == self.land_md5: headers['X-Land-Or-Sea'] = '1' elif md5sum == self.sea_md5: headers['X-Land-Or-Sea'] = '2' raise TheTileLeftANote(content=body, headers=headers)
def __call__(self, environ, start_response): key_morsel = Cookie(environ.get("HTTP_COOKIE", "")).get(self.toggle_key) # useful vars query = query_str2dict(environ.get("QUERY_STRING")) enable_by_cookie = key_morsel.value == self.enable_value if key_morsel else False enable_by_query = query.get(self.toggle_key) == self.enable_value # pop toggle_key from query dic to avoid case: '?_profile=on&_profile=' disable = query.pop(self.toggle_key, None) == "" # only can be disabled by query enable = not disable and (enable_by_query or enable_by_cookie) run_app, resp_body, saved_ss_args = self._intercept_call() # processing cookies and queries so = query.pop(self.SIMPLE_OUTPUT_TOGGLE_KEY, None) if so is not None: self.simple_output = so == "True" cookie_to_set = None if enable_by_query and not enable_by_cookie: cookie_to_set = "%s=%s; Path=/; HttpOnly" % (self.toggle_key, self.enable_value) elif disable: cookie_to_set = "%s=; Path=/; Max-Age=1; HttpOnly" % self.toggle_key if enable: start = time.time() profile = Profile() profile.runcall(run_app, environ) # here we call the WSGI app elapsed = time.time() - start else: profile = elapsed = None # for annoying IDE run_app(environ) status, headers = saved_ss_args[:2] headers_dic = Headers(headers) if cookie_to_set: headers_dic.add_header("Set-Cookie", cookie_to_set) # insert result into response content_type = headers_dic.get("Content-Type", "") if enable and status.startswith("200") and content_type.startswith("text/html"): environ["QUERY_STRING"] = dict2query_str(query) matched = _find_charset.match(content_type) encoding = matched.group(1) if matched else "ascii" rendered = self.render_result(profile, elapsed, environ).encode(encoding, "replace") resp_body = [insert_into_body(rendered, b"".join(resp_body))] headers_dic["Content-Length"] = str(len(resp_body[0])) start_response(status, headers, saved_ss_args[2] if len(saved_ss_args) == 3 else None) return resp_body
def get_static_file(self, path, url, stat_cache=None): # Optimization: bail early if file does not exist if stat_cache is None and not os.path.exists(path): raise MissingFileError(path) headers = Headers([]) self.add_mime_headers(headers, path, url) self.add_cache_headers(headers, path, url) if self.allow_all_origins: headers['Access-Control-Allow-Origin'] = '*' if self.add_headers_function: self.add_headers_function(headers, path, url) return StaticFile( path, headers.items(), stat_cache=stat_cache, encodings={ 'gzip': path + '.gz', 'br': path + '.br'})
def _parse_headers(environ): """ Parse the environmental variables, looking for HTTP request headers. :param environ: environmental variables :type environ: dict :return: request headers :rtype: dict """ headers = Headers([]) for key, value in environ.items(): match = _HTTP_HEADER_REGEX.match(key) if match is None: continue name = _normalize_header_name(match.group(0)) headers.add_header(name, value) return headers
def __init__(self, path, is_immutable, guess_type=mimetypes.guess_type, **config): self.path = path stat = os.stat(path) self.mtime_tuple = gmtime(stat.st_mtime) mimetype, encoding = guess_type(path) mimetype = mimetype or 'application/octet-stream' charset = self.get_charset(mimetype) params = {'charset': charset} if charset else {} self.headers = Headers([ ('Last-Modified', formatdate(stat.st_mtime, usegmt=True)), ('Content-Length', str(stat.st_size)), ]) self.headers.add_header('Content-Type', str(mimetype), **params) if encoding: self.headers['Content-Encoding'] = encoding max_age = self.FOREVER if is_immutable else config['max_age'] if max_age is not None: self.headers['Cache-Control'] = 'public, max-age=%s' % max_age if config['allow_all_origins']: self.headers['Access-Control-Allow-Origin'] = '*' gzip_path = path + self.GZIP_SUFFIX if os.path.isfile(gzip_path): self.gzip_path = gzip_path self.headers['Vary'] = 'Accept-Encoding' # Copy the headers and add the appropriate encoding and length self.gzip_headers = Headers(self.headers.items()) self.gzip_headers['Content-Encoding'] = 'gzip' self.gzip_headers['Content-Length'] = str(os.stat(gzip_path).st_size) else: self.gzip_path = self.gzip_headers = None
def bind(self): """ Clears old data and creates a brand new Response object """ self._COOKIES = None self.status = 200 self.header_list = [] self.header = HeaderWrapper(self.header_list) self.content_type = 'text/html' self.error = None
def __init__(self, body=b'', status=None, headers=None): self.headers = Headers() self._body = body self._status_code = status or self.default_status self._cookies = SimpleCookie() if headers: for name, value in headers.items(): self.headers.add_header(name, value)
def newapp(environ, start_response): body = app(environ, capture) status = resp['status'] headers = Headers(resp['headers']) already = 'Content-Encoding' in headers accepted = 'gzip' in environ.get('HTTP_ACCEPT_ENCODING', '') if not accepted or already: # no compress start_response(status, list(headers.items())) return body content = gzip.compress(b''.join(body)) if hasattr(body, 'close'): body.close() headers['Content-Encoding'] = 'gzip' start_response(status, list(headers.items())) return [content]
class Response(threading.local): """ Represents a single response using thread-local namespace. """ def bind(self): """ Clears old data and creates a brand new Response object """ self._COOKIES = None self.status = 200 self.header_list = [] self.header = HeaderWrapper(self.header_list) self.content_type = "text/html" self.error = None self.charset = "utf8" def wsgiheaders(self): """ Returns a wsgi conform list of header/value pairs """ for c in self.COOKIES.itervalues(): self.header.add_header("Set-Cookie", c.OutputString()) return [(h.title(), str(v)) for h, v in self.header.items()] @property def COOKIES(self): if not self._COOKIES: self._COOKIES = SimpleCookie() return self._COOKIES def set_cookie(self, key, value, **kargs): """ Sets a Cookie. Optional settings: expires, path, comment, domain, max-age, secure, version, httponly """ self.COOKIES[key] = value for k, v in kargs.iteritems(): self.COOKIES[key][k] = v def get_content_type(self): """ Get the current 'Content-Type' header. """ return self.header["Content-Type"] def set_content_type(self, value): if "charset=" in value: self.charset = value.split("charset=")[-1].split(";")[0].strip() self.header["Content-Type"] = value content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__)
def bind(self, app): """ Clears old data and creates a brand new Response object """ self._COOKIES = None self.status = 200 self.header_list = [] self.header = HeaderWrapper(self.header_list) self.charset = "UTF-8" self.content_type = "text/html; charset=UTF-8" self.error = None self.app = app
def newapp(environ, start_response): raw = app(environ, capture) status = resp['status'] headers = Headers(resp['headers']) if (not 'Last-Modified' in headers or not environ.get('HTTP_IF_MODIFIED_SINCE')): start_response(status, list(headers.items())) return raw last_m = eutils.parsedate(headers['Last-Modified']) since_m = eutils.parsedate(environ['HTTP_IF_MODIFIED_SINCE']) if since_m < last_m: start_response(status, list(headers.items())) return raw else: start_response('304 Not Modified', list(headers.items())) if hasattr(raw, 'close'): raw.close() return [b'']
def board_app(env, resp): path = env['PATH_INFO'] m = board_re.match(path) board = m.group(1) message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja')) headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')]) resp("200 OK", headers.items()) html = [ '<!DOCTYPE html>', '<html><head>', '<meta http-equiv="content-type" content="text/html; charset=Shift_JIS">', '<title>%s - %s</title>' % (message['logo'], message['description']), '<meta name="description" content="%s - %s">' % (message['logo'], message['description']), '</head><body>', '<h1>%s - %s</h1>' % (message['logo'], message['description']), '</body></html>', ] return ((c + '\n').encode('sjis', 'ignore') for c in html)
def finish_header(self): self.file = BytesIO() self.headers = Headers(self.headerlist) cdis = self.headers.get('Content-Disposition', '') ctype = self.headers.get('Content-Type', '') if not cdis: raise MultipartError('Content-Disposition header is missing.') self.disposition, self.options = parse_options_header(cdis) self.name = self.options.get('name') self.filename = self.options.get('filename') self.content_type, options = parse_options_header(ctype) self.charset = options.get('charset') or self.charset self.content_length = int(self.headers.get('Content-Length', '-1'))
class Response(threading.local): """ Represents a single response using thread-local namespace. """ def bind(self): """ Clears old data and creates a brand new Response object """ self._COOKIES = None self.status = 200 self.header_list = [] self.header = HeaderWrapper(self.header_list) self.content_type = 'text/html' self.error = None def wsgiheaders(self): ''' Returns a wsgi conform list of header/value pairs ''' for c in self.COOKIES.itervalues(): self.header.add_header('Set-Cookie', c.OutputString()) return [(h.title(), str(v)) for h, v in self.header_list] @property def COOKIES(self): if not self._COOKIES: self._COOKIES = SimpleCookie() return self._COOKIES def set_cookie(self, key, value, **kargs): """ Sets a Cookie. Optional settings: expires, path, comment, domain, max-age, secure, version, httponly """ self.COOKIES[key] = value for k in kargs: self.COOKIES[key][k] = kargs[k] def get_content_type(self): '''Gives access to the 'Content-Type' header and defaults to 'text/html'.''' return self.header['Content-Type'] def set_content_type(self, value): self.header['Content-Type'] = value content_type = property(get_content_type, set_content_type, None, get_content_type.__doc__)
def finish_header(self): self.file = BytesIO() self.headers = Headers(self.headerlist) cdis = self.headers.get("Content-Disposition", "") ctype = self.headers.get("Content-Type", "") clen = self.headers.get("Content-Length", "-1") if not cdis: raise MultipartError("Content-Disposition header is missing.") self.disposition, self.options = parse_options_header(cdis) self.name = self.options.get("name") self.filename = self.options.get("filename") self.content_type, options = parse_options_header(ctype) self.charset = options.get("charset") or self.charset self.content_length = int(self.headers.get("Content-Length", "-1"))
class Response(object): """ Wrapper class around the start_response and return iterable of the WSGI protocol. :ivar status: HTTP status of the response :type status: int :ivar headers: response headers :type headers: dict :ivar cookies: cookies to set :type cookies: dict """ def __init__(self): self.status = None self.__header_list = [] self.headers = Headers(self.__header_list) self.headers['date'] = rfc1123_date() self.cookies = SimpleCookie() def start_response_args(self, body): """ Return a status string and a list of headers for this response, appropriate as arguments to the start_response function. :param body: response body :type body: str :return: tuple of status string and list of header tuples :rtype: (str, list) """ if self.status is None: raise RuntimeError('response status was not set') self.headers['content-length'] = str(len(body)) for cookie in self.cookies: self.headers.add_header('set-cookie', cookie.output(header='')) status_str = '%d %s' % (self.status, httplib.responses[self.status]) return (status_str, self.__header_list)
def finish_header(self): self.file = BytesIO() self.headers = Headers(self.headerlist) cdis = self.headers.get('Content-Disposition','') ctype = self.headers.get('Content-Type','') clen = self.headers.get('Content-Length','-1') if not cdis: raise MultipartError('Content-Disposition header is missing.') self.disposition, self.options = parse_options_header(cdis) self.name = self.options.get('name') self.filename = self.options.get('filename') self.content_type, options = parse_options_header(ctype) self.charset = options.get('charset') or self.charset self.content_length = int(self.headers.get('Content-Length','-1')) self.content_transfer_encoding = self.headers.get('Content-Transfer-Encoding') if self.content_transfer_encoding not in [None, 'base64', 'quoted-printable']: raise MultipartError('invalid Content-Transfer-Encoding')
def testExtras(self): h = Headers([]) self.assertEqual(str(h), "\r\n") h.add_header("foo", "bar", baz="spam") self.assertEqual(h["foo"], 'bar; baz="spam"') self.assertEqual(str(h), 'foo: bar; baz="spam"\r\n\r\n') h.add_header("Foo", "bar", cheese=None) self.assertEqual(h.get_all("foo"), ['bar; baz="spam"', "bar; cheese"]) self.assertEqual(str(h), 'foo: bar; baz="spam"\r\n' "Foo: bar; cheese\r\n" "\r\n")
def testExtras(self): h = Headers([]) self.assertEqual(str(h),'\r\n') h.add_header('foo','bar',baz="spam") self.assertEqual(h['foo'], 'bar; baz="spam"') self.assertEqual(str(h),'foo: bar; baz="spam"\r\n\r\n') h.add_header('Foo','bar',cheese=None) self.assertEqual(h.get_all('foo'), ['bar; baz="spam"', 'bar; cheese']) self.assertEqual(str(h), 'foo: bar; baz="spam"\r\n' 'Foo: bar; cheese\r\n' '\r\n' )
class MultipartPart(object): def __init__(self, buffer_size=2**16, memfile_limit=2**18, charset="latin1"): self.headerlist = [] self.headers = None self.file = False self.size = 0 self._buf = b"" self.disposition = None self.name = None self.filename = None self.content_type = None self.charset = charset self.memfile_limit = memfile_limit self.buffer_size = buffer_size def feed(self, line, nl=""): if self.file: return self.write_body(line, nl) return self.write_header(line, nl) def write_header(self, line, nl): line = line.decode(self.charset) if not nl: raise MultipartError("Unexpected end of line in header.") if not line.strip(): # blank line -> end of header segment self.finish_header() elif line[0] in " \t" and self.headerlist: name, value = self.headerlist.pop() self.headerlist.append((name, value + line.strip())) else: if ":" not in line: raise MultipartError("Syntax error in header: No colon.") name, value = line.split(":", 1) self.headerlist.append((name.strip(), value.strip())) def write_body(self, line, nl): if not line and not nl: return # This does not even flush the buffer self.size += len(line) + len(self._buf) self.file.write(self._buf + line) self._buf = nl if self.content_length > 0 and self.size > self.content_length: raise MultipartError("Size of body exceeds Content-Length header.") if self.size > self.memfile_limit and isinstance(self.file, BytesIO): # TODO: What about non-file uploads that exceed the memfile_limit? self.file, old = TemporaryFile(mode="w+b"), self.file old.seek(0) copy_file(old, self.file, self.size, self.buffer_size) def finish_header(self): self.file = BytesIO() self.headers = Headers(self.headerlist) content_disposition = self.headers.get("Content-Disposition", "") content_type = self.headers.get("Content-Type", "") if not content_disposition: raise MultipartError("Content-Disposition header is missing.") self.disposition, self.options = parse_options_header( content_disposition) self.name = self.options.get("name") self.filename = self.options.get("filename") self.content_type, options = parse_options_header(content_type) self.charset = options.get("charset") or self.charset self.content_length = int(self.headers.get("Content-Length", "-1")) def is_buffered(self): """ Return true if the data is fully buffered in memory.""" return isinstance(self.file, BytesIO) @property def value(self): """ Data decoded with the specified charset """ return self.raw.decode(self.charset) @property def raw(self): """ Data without decoding """ pos = self.file.tell() self.file.seek(0) try: val = self.file.read() except IOError: raise finally: self.file.seek(pos) return val def save_as(self, path): with open(path, "wb") as fp: pos = self.file.tell() try: self.file.seek(0) size = copy_file(self.file, fp) finally: self.file.seek(pos) return size def close(self): if self.file: self.file.close() self.file = False
def _start_response(self, status: str, response_headers: List[Any]): self._status = status self._headers = Headers(response_headers) # type: ignore self._status_code = int(self._status.split(' ')[0]) # 200 OK
class SimDistribHandler(object): def __init__(self, archive, wsgienv, start_resp): self.arch = archive self._env = wsgienv self._start = start_resp self._meth = wsgienv.get('REQUEST_METHOD', 'GET') self._hdr = Headers([]) self._code = 0 self._msg = "unknown status" def send_error(self, code, message): status = "{0} {1}".format(str(code), message) self._start(status, [], sys.exc_info()) return [] def add_header(self, name, value): self._hdr.add_header(name, value) def set_response(self, code, message): self._code = code self._msg = message def end_headers(self): status = "{0} {1}".format(str(self._code), self._msg) self._start(status, self._hdr.items()) def handle(self, env, start_resp): meth_handler = 'do_' + self._meth path = self._env.get('PATH_INFO', '/')[1:] params = cgi.parse_qs(self._env.get('QUERY_STRING', '')) if hasattr(self, meth_handler): return getattr(self, meth_handler)(path, params) else: return self.send_error( 403, self._meth + " not supported on this resource") def do_HEAD(self, path, params=None, forhead=False): return self.do_GET(path, params, True) def do_GET(self, path, params=None, forhead=False): aid = None vers = None path = path.strip('/') if path.startswith("od/ds/"): path = path[len("od/ds/"):] print("processing " + path) # refresh the archive self.arch.loadinfo() if not path: try: out = json.dumps(self.arch.aipids) + '\n' except Exception as ex: return self.send_error(500, "Internal error") self.set_response(200, "AIP Identifiers") self.add_header('Content-Type', 'application/json') self.add_header('Content-Length', str(len(out))) self.end_headers() if forhead: return [] return [out] elif path.startswith("_aip/"): # requesting a bag file path = path[len("_aip/"):].strip('/') filepath = os.path.join(self.arch.dir, path) if os.path.isfile(filepath): self.set_response(200, "Bag file found") self.add_header('Content-Type', "application/zip") self.end_headers() if forhead: return [] return self.iter_file(filepath) else: return self.send_error(404, "bag file does not exist") elif '/' in path: parts = path.split('/', 1) aid = parts[0] path = (len(parts) > 1 and parts[1]) or '' print("accessing " + aid) elif path: aid = path path = '' else: return self.send_error(404, "resource does not exist") # path-info is now captured as aid and path if aid not in self.arch._aips: return self.send_error(404, "resource does not exist") if not path: self.set_response(200, "AIP Identifier exists") self.add_header('Content-Type', 'application/json') self.add_header('Content-Length', str(len(aid) + 4)) self.end_headers() if forhead: return [] return ['["' + aid + '"]'] elif path == "_aip": try: out = json.dumps(self.arch.list_bags(aid)) + '\n' except Exception, ex: return self.send_error(500, "Internal error") self.set_response(200, "All bags for ID") self.add_header('Content-Type', 'application/json') self.add_header('Content-Length', str(len(out))) self.end_headers() if forhead: return [] return [out] elif path == "_aip/_head": try: out = self.arch.head_for(aid) if out: out = json.dumps(out) + '\n' except Exception, ex: print( "Failed to create JSON output for head bag, aid={0}: {2}". format(aid, vers, str(ex))) return self.send_error(500, "Internal error")
def __init__(self, path): self.path = path self.headers = Headers([])
class SimIngestHandler(object): def __init__(self, wsgienv, start_resp): self._env = wsgienv self._start = start_resp self._meth = wsgienv.get('REQUEST_METHOD', 'GET') self._hdr = Headers([]) self._code = 0 self._msg = "unknown status" self._auth = (authmeth, authkey) def send_error(self, code, message): status = "{0} {1}".format(str(code), message) self._start(status, [], sys.exc_info()) return [] def add_header(self, name, value): self._hdr.add_header(name, value) def set_response(self, code, message): self._code = code self._msg = message def end_headers(self): status = "{0} {1}".format(str(self._code), self._msg) self._start(status, self._hdr.items()) def handle(self, env, start_resp): meth_handler = 'do_' + self._meth path = self._env.get('PATH_INFO', '/')[1:] params = cgi.parse_qs(self._env.get('QUERY_STRING', '')) print("AUTH METHOD: %s" % self._auth[0], file=sys.stderr) if not self.authorize(): return self.send_unauthorized() if hasattr(self, meth_handler): return getattr(self, meth_handler)(path, params) else: return self.send_error( 403, self._meth + " not supported on this resource") def authorize(self): if self._auth[0] == 'header': return self.authorize_via_headertoken() else: return self.authorize_via_queryparam() def authorize_via_queryparam(self): params = cgi.parse_qs(self._env.get('QUERY_STRING', '')) auths = params.get('auth', []) if self._auth[1]: # match the last value provided return len(auths) > 0 and self._auth[1] == auths[-1] if len(auths) > 0: log.warn( "Authorization key provided, but none has been configured") return len(auths) == 0 def authorize_via_headertoken(self): authhdr = self._env.get('HTTP_AUTHORIZATION', "") print("Request HTTP_AUTHORIZATION: %s" % authhdr, file=sys.stderr) parts = authhdr.split() if self._auth[1]: return len(parts) > 1 and parts[0] == "Bearer" and \ self._auth[1] == parts[1] if authhdr: log.warn( "Authorization key provided, but none has been configured") return authhdr == "" def send_unauthorized(self): self.set_response(401, "Not authorized") if self._auth[0] == 'header': self.add_header('WWW-Authenticate', 'Bearer') self.end_headers() return [] def do_GET(self, path, params=None): path = path.strip('/') if not path: try: out = json.dumps(["nerdm", "invalid"]) + '\n' except Exception, ex: return self.send_error(500, "Internal error") self.set_response(200, "Supported Record Types") self.add_header('Content-Type', 'application/json') self.end_headers() return [out] elif path in "nerdm invalid".split(): self.set_response(200, "Service is ready") self.add_header('Content-Type', 'application/json') self.end_headers() return ["Service ready\n"]
def test_auth_ok(static_auth): request = mock.Mock() request.headers = Headers([("authorization", "Bearer test")]) assert static_auth.is_authorized(request)
TLDLESS_NETLOC_RESOLVER = { 'ketab': ketabir_scr, 'noorlib': noorlib_scr, 'noormags': noormags_scr, 'web.archive': waybackmachine_scr, 'web-beta.archive': waybackmachine_scr, 'books.google.co': googlebooks_scr, 'books.google.com': googlebooks_scr, 'books.google': googlebooks_scr, 'google': google_encrypted_scr, 'encrypted.google': google_encrypted_scr, }.get RESPONSE_HEADERS = Headers([('Content-Type', 'text/html; charset=UTF-8')]) getLogger('requests').setLevel(WARNING) getLogger('langid').setLevel(WARNING) def get_root_logger(): custom_logger = getLogger() custom_logger.setLevel(INFO) srcdir = dirname(__file__) handler = RotatingFileHandler(filename=f'{srcdir}/citer.log', mode='a', maxBytes=20000, backupCount=0, encoding='utf-8', delay=0)
'application/x-jpg': '.jpg', 'application/x-bmp': '.bmp', 'application/msword': '.doc', '': '', } return dict[type] if type in dict.keys() else '' DEFAULT_MAX_THREAD = 5 DEFAULT_MAX_CONNECTIONS = 16 HEADERS_CHROME = Headers([ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36' ), ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8' ), ('Accept-Encoding', 'gzip, deflate, br'), ('Accept-Language', 'zh-CN,zh;q=0.9') ]) class UrlPool(Packer, object): def __init__(self, parent, max_retry=-1, max_conn=DEFAULT_MAX_CONNECTIONS, max_speed=-1): self.parent = parent self.list = [] self.dict = {}
def newapp(environ, start_response): raw = app(environ, capture) status = resp['status'] headers = Headers(resp['headers']) headers.setdefault('Accept-Range', 'bytes') range = environ.get('HTTP_RANGE') if (range is None or ',' in range # not deal with multi-part range or not status.startswith('2')): # not success status start_response(status, list(headers.items())) return raw def error_416(): start_response('416 Requested Range Not Satisfiable', list(headers.items())) if hasattr(raw, 'close'): raw.close() return [b''] m = re.match(r'bytes=([0-9]+)?-([0-9]+)?', range) if not m or (not m.group(1) and not m.group(2)): return error_416() content = b''.join(raw) begin = int(m.group(1)) if m.group(1) else None end = int(m.group(2)) if m.group(2) else None # because 0 is False has_begin = begin is not None has_end = end is not None if (has_begin and has_end) and end < begin: return error_416() if has_end and len(content) <= end: return error_416() if has_begin and len(content) <= begin: return error_416() if has_begin and has_end: # bytes=begin-end c_range = 'bytes {}-{}/{}'.format(begin, end, len(content)) body = content[begin:end + 1] elif has_begin: # bytes=begin- c_range = 'bytes {}-{}/{}'.format(begin, len(content) - 1, len(content)) body = content[begin:] else: # bytes=-end c_range = 'bytes {}-{}/{}'.format( len(content) - end, len(content) - 1, len(content)) body = content[len(content) - end:] headers['Content-Range'] = c_range start_response('206 Partial Content', list(headers.items())) if hasattr(raw, 'close'): raw.close() return [body]
def getPreview(layer): """ Get a type string and dynamic map viewer HTML for a given layer. """ return 200, Headers([('Content-Type', 'text/html')]), Core._preview(layer)
def testMappingInterface(self): test = [('x','y')] self.assertEqual(len(Headers()), 0) self.assertEqual(len(Headers([])),0) self.assertEqual(len(Headers(test[:])),1) self.assertEqual(Headers(test[:]).keys(), ['x']) self.assertEqual(Headers(test[:]).values(), ['y']) self.assertEqual(Headers(test[:]).items(), test) self.assertIsNot(Headers(test).items(), test) # must be copy! h = Headers() del h['foo'] # should not raise an error h['Foo'] = 'bar' for m in h.__contains__, h.get, h.get_all, h.__getitem__: self.assertTrue(m('foo')) self.assertTrue(m('Foo')) self.assertTrue(m('FOO')) self.assertFalse(m('bar')) self.assertEqual(h['foo'],'bar') h['foo'] = 'baz' self.assertEqual(h['FOO'],'baz') self.assertEqual(h.get_all('foo'),['baz']) self.assertEqual(h.get("foo","whee"), "baz") self.assertEqual(h.get("zoo","whee"), "whee") self.assertEqual(h.setdefault("foo","whee"), "baz") self.assertEqual(h.setdefault("zoo","whee"), "whee") self.assertEqual(h["foo"],"baz") self.assertEqual(h["zoo"],"whee")
def requestHandler2(config_hint, path_info, query_string=None, script_name=''): """ Generate a set of headers and response body for a given request. TODO: Replace requestHandler() with this function in TileStache 2.0.0. Requires a configuration and PATH_INFO (e.g. "/example/0/0/0.png"). Config_hint parameter can be a path string for a JSON configuration file or a configuration object with 'cache', 'layers', and 'dirpath' properties. Query string is optional, currently used for JSON callbacks. Calls Layer.getTileResponse() to render actual tiles, and getPreview() to render preview.html. """ headers = Headers([]) try: # ensure that path_info is at least a single "/" path_info = '/' + (path_info or '').lstrip('/') layer = requestLayer(config_hint, path_info) query = parse_qs(query_string or '') try: callback = query['callback'][0] except KeyError: callback = None # # Special case for index page. # if path_info == '/': mimetype, content = getattr( layer.config, 'index', ('text/plain', 'TileStache says hello.')) return 200, Headers([('Content-Type', mimetype)]), content coord, extension = splitPathInfo(path_info)[1:] if extension == 'html' and coord is None: status_code, headers, content = getPreview(layer) elif extension.lower() in layer.redirects: other_extension = layer.redirects[extension.lower()] redirect_uri = script_name redirect_uri += mergePathInfo(layer.name(), coord, other_extension) if query_string: redirect_uri += '?' + query_string headers['Location'] = redirect_uri headers['Content-Type'] = 'text/plain' return 302, headers, 'You are being redirected to %s\n' % redirect_uri else: status_code, headers, content = layer.getTileResponse( coord, extension) if layer.allowed_origin: headers.setdefault('Access-Control-Allow-Origin', layer.allowed_origin) if callback and 'json' in headers['Content-Type']: headers['Content-Type'] = 'application/javascript; charset=utf-8' content = '%s(%s)' % (callback, content) if layer.max_cache_age is not None: expires = datetime.utcnow() + timedelta( seconds=layer.max_cache_age) headers.setdefault('Expires', expires.strftime('%a, %d %b %Y %H:%M:%S GMT')) headers.setdefault('Cache-Control', 'public, max-age=%d' % layer.max_cache_age) except Core.KnownUnknown as e: out = StringIO() print('Known unknown!', file=out) print(e, file=out) print('', file=out) print('\n'.join(Core._rummy()), file=out) headers['Content-Type'] = 'text/plain' status_code, content = 500, out.getvalue().encode('ascii') return status_code, headers, content
def __getitem__(self, name): ret = Headers.__getitem__(self, name) if ret is None: return '' else: return str(ret)
def __call__(self, environ, start_response): '''Main function for handling a single request. Follows the WSGI API. @param environ: dictionary with environment variables for the request and some special variables. See the PEP for expected variables. @param start_response: a function that can be called to set the http response and headers. For example:: start_response(200, [('Content-Type', 'text/plain')]) @returns: the html page content as a list of lines ''' headerlist = [] headers = Headers(headerlist) path = environ.get('PATH_INFO', '/') try: methods = ('GET', 'HEAD') if not environ['REQUEST_METHOD'] in methods: raise WWWError('405', headers=[('Allow', ', '.join(methods))]) # cleanup path #~ print 'INPUT', path path = path.replace('\\', '/') # make it windows save isdir = path.endswith('/') parts = [p for p in path.split('/') if p and not p == '.'] if [p for p in parts if p.startswith('.')]: # exclude .. and all hidden files from possible paths raise PathNotValidError() path = '/' + '/'.join(parts) if isdir and not path == '/': path += '/' #~ print 'PATH', path if not path: path = '/' elif path == '/favicon.ico': path = '/+resources/favicon.ico' else: path = urllib.unquote(path) if path == '/': headers.add_header('Content-Type', 'text/html', charset='utf-8') content = self.render_index() elif path.startswith('/+docs/'): dir = self.notebook.document_root if not dir: raise PageNotFoundError(path) file = dir.file(path[7:]) content = [file.raw()] # Will raise FileNotFound when file does not exist headers['Content-Type'] = file.get_mimetype() elif path.startswith('/+file/'): file = self.notebook.dir.file(path[7:]) # TODO: need abstraction for getting file from top level dir ? content = [file.raw()] # Will raise FileNotFound when file does not exist headers['Content-Type'] = file.get_mimetype() elif path.startswith('/+resources/'): if self.template.resources_dir: file = self.template.resources_dir.file(path[12:]) if not file.exists(): file = data_file('pixmaps/%s' % path[12:]) else: file = data_file('pixmaps/%s' % path[12:]) if file: content = [file.raw()] # Will raise FileNotFound when file does not exist headers['Content-Type'] = file.get_mimetype() else: raise PageNotFoundError(path) else: # Must be a page or a namespace (html file or directory path) headers.add_header('Content-Type', 'text/html', charset='utf-8') if path.endswith('.html'): pagename = path[:-5].replace('/', ':') elif path.endswith('/'): pagename = path[:-1].replace('/', ':') else: raise PageNotFoundError(path) path = self.notebook.resolve_path(pagename) page = self.notebook.get_page(path) if page.hascontent: content = self.render_page(page) elif page.haschildren: content = self.render_index(page) else: raise PageNotFoundError(page) except Exception, error: headerlist = [] headers = Headers(headerlist) headers.add_header('Content-Type', 'text/plain', charset='utf-8') if isinstance(error, (WWWError, FileNotFoundError)): logger.error(error.msg) if isinstance(error, FileNotFoundError): error = PageNotFoundError(path) # show url path instead of file path if error.headers: for key, value in error.headers: headers.add_header(key, value) start_response(error.status, headerlist) content = unicode(error).splitlines(True) # TODO also handle template errors as special here else: # Unexpected error - maybe a bug, do not expose output on bugs # to the outside world logger.exception('Unexpected error:') start_response('500 Internal Server Error', headerlist) content = ['Internal Server Error'] if environ['REQUEST_METHOD'] == 'HEAD': return [] else: return [string.encode('utf-8') for string in content]
def test_no_auth_header(static_auth): request = mock.Mock() request.headers = Headers([]) assert not static_auth.is_authorized(request)
def update(self, url=None, headers=None): if url: self.load(url) if headers: self.headers = Headers(headers)
class BaseResponse: """Base class for Response.""" default_status = 200 default_content_type = 'text/plain;' def __init__(self, body=None, status=None, headers=None): self._body = body if body else [b''] self._status_code = status or self.default_status self.headers = Headers() self._cookies = SimpleCookie() if headers: for name, value in headers.items(): self.headers.add_header(name, value) @property def body(self): return self._body @property def status_code(self): """ The HTTP status code as an integer (e.g. 404).""" return self._status_code @property def status(self): """ The HTTP status line as a string (e.g. ``404 Not Found``).""" status = _HTTP_STATUS_LINES.get(self._status_code) return str(status or ('{} Unknown'.format(self._status_code))) @status.setter def status(self, status_code): if not 100 <= status_code <= 999: raise ValueError('Status code out of range.') self._status_code = status_code @property def headerlist(self): """ WSGI conform list of (header, value) tuples. """ if 'Content-Type' not in self.headers: self.headers.add_header('Content-Type', self.default_content_type) if self._cookies: for c in self._cookies.values(): self.headers.add_header('Set-Cookie', c.OutputString()) return self.headers.items() def set_cookie(self, key, value, expires=None, max_age=None, path='/', secret=None, digestmod=hashlib.sha256): from kobin.app import current_config if secret is None: secret = current_config('SECRET_KEY') if secret: if isinstance(secret, str): secret = secret.encode('utf-8') encoded = base64.b64encode( pickle.dumps((key, value), pickle.HIGHEST_PROTOCOL)) sig = base64.b64encode( hmac.new(secret, encoded, digestmod=digestmod).digest()) value_bytes = b'!' + sig + b'?' + encoded value = value_bytes.decode('utf-8') self._cookies[key] = value if len(key) + len(value) > 3800: raise ValueError('Content does not fit into a cookie.') if max_age is not None: if isinstance(max_age, int): max_age_value = max_age else: max_age_value = max_age.seconds + max_age.days * 24 * 3600 self._cookies[key]['max-age'] = max_age_value if expires is not None: if isinstance(expires, int): expires_value = expires else: expires_value = time.strftime("%a, %d %b %Y %H:%M:%S GMT", expires.timetuple()) self._cookies[key]['expires'] = expires_value if path: self._cookies[key]['path'] = path def delete_cookie(self, key, **kwargs): kwargs['max_age'] = -1 kwargs['expires'] = 0 self.set_cookie(key, '', **kwargs)
class Response: default_status = 200 default_content_type = 'text/html; charset=UTF-8' def __init__(self, body: str='', status: int=None, headers: Dict=None, **more_headers) -> None: self.headers = Headers() self.body = body self._status_code = status or self.default_status self._cookies = SimpleCookie() # type: ignore if headers: for name, value in headers.items(): self.headers.add_header(name, value) if more_headers: for name, value in more_headers.items(): self.headers.add_header(name, value) @property def status_code(self): """ The HTTP status code as an integer (e.g. 404).""" return self._status_code @property def status(self): """ The HTTP status line as a string (e.g. ``404 Not Found``).""" if not 100 <= self._status_code <= 999: raise ValueError('Status code out of range.') status = _HTTP_STATUS_LINES.get(self._status_code) return str(status or ('{} Unknown'.format(self._status_code))) @status.setter def status(self, status_code: int): if not 100 <= status_code <= 999: raise ValueError('Status code out of range.') self._status_code = status_code @property def headerlist(self) -> List[Tuple[str, str]]: """ WSGI conform list of (header, value) tuples. """ out = [] # type: List[Tuple[str, str]] if 'Content-Type' not in self.headers: self.headers.add_header('Content-Type', self.default_content_type) out += [(key, value) for key in self.headers.keys() for value in self.headers.get_all(key)] if self._cookies: for c in self._cookies.values(): out.append(('Set-Cookie', c.OutputString())) return [(k, v.encode('utf8').decode('latin1')) for (k, v) in out] def set_cookie(self, key: str, value: Any, expires: str=None, path: str=None, **options: Dict[str, Any]) -> None: from datetime import timedelta, datetime, date import time self._cookies[key] = value if expires: self._cookies[key]['expires'] = expires if path: self._cookies[key]['path'] = path for k, v in options.items(): if k == 'max_age': if isinstance(v, timedelta): v = v.seconds + v.days * 24 * 3600 # type: ignore if k == 'expires': if isinstance(v, (date, datetime)): v = v.timetuple() # type: ignore elif isinstance(v, (int, float)): v = v.gmtime(value) # type: ignore v = time.strftime("%a, %d %b %Y %H:%M:%S GMT", v) # type: ignore self._cookies[key][k.replace('_', '-')] = v # type: ignore def delete_cookie(self, key, **kwargs) -> None: kwargs['max_age'] = -1 kwargs['expires'] = 0 self.set_cookie(key, '', **kwargs) def apply(self, other): self.status = other._status_code self._cookies = other._cookies self.headers = other.headers self.body = other.body
def __init__(self: 'Response', start_response, request: Request): self.request = request self.id = request.id self._start_response = start_response self.headers = Headers() self.status_code = 200
def run_app(self, conn): self.status = "200 OK" self.size = 0 self.expires = None self.etag = None self.content_type = 'text/plain' self.content_length = None if __debug__: self.err_log.debug('Getting sock_file') # Build our file-like object sock_file = conn.makefile('rb', BUF_SIZE) request = self.read_request_line(sock_file) if request['method'].upper() not in ('GET', ): self.status = "501 Not Implemented" try: # Get our file path headers = dict([(str(k.lower()), v) for k, v in self.read_headers(sock_file).items()]) rpath = request.get('path', '').lstrip('/') filepath = os.path.join(self.root, rpath) filepath = os.path.abspath(filepath) if __debug__: self.err_log.debug('Request for path: %s' % filepath) self.closeConnection = headers.get('connection', 'close').lower() == 'close' self.headers = Headers([ ('Date', formatdate(usegmt=True)), ('Server', HTTP_SERVER_SOFTWARE), ('Connection', headers.get('connection', 'close')), ]) if not filepath.lower().startswith(self.root.lower()): # File must be within our root directory self.status = "400 Bad Request" self.closeConnection = True elif not os.path.exists(filepath): self.status = "404 File Not Found" self.closeConnection = True elif os.path.isdir(filepath): self.serve_dir(filepath, rpath) elif os.path.isfile(filepath): self.serve_file(filepath, headers) else: # It exists but it's not a file or a directory???? # What is it then? self.status = "501 Not Implemented" self.closeConnection = True h = self.headers statcode, statstr = self.status.split(' ', 1) statcode = int(statcode) if statcode >= 400: h.add_header('Content-Type', self.content_type) self.data = [statstr] # Build our output headers header_data = HEADER_RESPONSE % (self.status, str(h)) # Send the headers if __debug__: self.err_log.debug('Sending Headers: %s' % repr(header_data)) self.conn.sendall(b(header_data)) for data in self.data: self.conn.sendall(b(data)) if hasattr(self.data, 'close'): self.data.close() finally: if __debug__: self.err_log.debug('Finally closing sock_file') sock_file.close()
class Handler(object): badidre = re.compile(r"[<>\s]") def __init__(self, service, siptype, wsgienv, start_resp, auth=None): self._svc = service self._env = wsgienv self._start = start_resp self._meth = wsgienv.get('REQUEST_METHOD', 'GET') self._hdr = Headers([]) self._code = 0 self._msg = "unknown status" self._auth = auth def send_error(self, code, message): stat = "{0} {1}".format(str(code), message) self._start(stat, [], sys.exc_info()) def add_header(self, name, value): # Caution: HTTP does not support Unicode characters (see # https://www.python.org/dev/peps/pep-0333/#unicode-issues); # thus, this will raise a UnicodeEncodeError if the input strings # include Unicode (char code > 255). e = "ISO-8859-1" self._hdr.add_header(name.encode(e), value.encode(e)) def set_response(self, code, message): self._code = code self._msg = message def end_headers(self): stat = "{0} {1}".format(str(self._code), self._msg) self._start(stat, self._hdr.items()) def handle(self): meth_handler = 'do_' + self._meth path = self._env.get('PATH_INFO', '/').strip('/') if not self.authorize(): return self.send_unauthorized() if hasattr(self, meth_handler): out = getattr(self, meth_handler)(path) if isinstance(out, list) and len(out) > 0: out.append('\n') return out else: return self.send_error( 403, self._meth + " not supported on this resource") def authorize(self): if self._auth[0] == 'header': return self.authorize_via_headertoken() else: return self.authorize_via_queryparam() def authorize_via_queryparam(self): params = cgi.parse_qs(self._env.get('QUERY_STRING', '')) auths = params.get('auth', []) if self._auth[1]: # match the last value provided return len(auths) > 0 and self._auth[1] == auths[-1] if len(auths) > 0: log.warn( "Authorization key provided, but none has been configured") return len(auths) == 0 def authorize_via_headertoken(self): authhdr = self._env.get('HTTP_AUTHORIZATION', "") parts = authhdr.split() if self._auth[1]: return len(parts) > 1 and parts[0] == "Bearer" and \ self._auth[1] == parts[1] if authhdr: log.warn( "Authorization key provided, but none has been configured") return authhdr == "" def send_unauthorized(self): self.set_response(401, "Not authorized") if self._auth[0] == 'header': self.add_header('WWW-Authenticate', 'Bearer') self.end_headers() return [] def do_GET(self, path): # return the status on request or a list of previous requests steps = path.split('/') if steps[0] == '': try: out = json.dumps(['midas']) except Exception, ex: log.exception("Internal error: " + str(ex)) self.send_error(500, "Internal error") return ["[]"] self.set_response(200, "Supported SIP Types") self.add_header('Content-Type', 'application/json') self.end_headers() return [out] elif steps[0] == 'midas': if len(steps) > 2: path = '/'.join(steps[1:]) self.send_error(400, "Unsupported SIP identifier: " + path) return [] elif len(steps) > 1: if steps[1].startswith("_") or steps[1].startswith(".") or \ self.badidre.search(steps[1]): self.send_error(400, "Unsupported SIP identifier: " + path) return [] return self.request_status(steps[1]) else: return self.requests()
def run_app(self, conn): self.size = 0 self.header_set = Headers([]) self.headers_sent = False self.error = (None, None) self.chunked = False sections = None output = None if __debug__: self.err_log.debug('Getting sock_file') # Build our file-like object if PY3K: sock_file = conn.makefile(mode='rb', buffering=BUF_SIZE) else: sock_file = conn.makefile(BUF_SIZE) try: # Read the headers and build our WSGI environment self.environ = environ = self.build_environ(sock_file, conn) # Handle 100 Continue if environ.get('HTTP_EXPECT', '') == '100-continue': res = environ['SERVER_PROTOCOL'] + ' 100 Continue\r\n\r\n' conn.sendall(b(res)) # Send it to our WSGI application output = self.app(environ, self.start_response) if not hasattr(output, '__len__') and not hasattr( output, '__iter__'): self.error = ('500 Internal Server Error', 'WSGI applications must return a list or ' 'generator type.') if hasattr(output, '__len__'): sections = len(output) for data in output: # Don't send headers until body appears if data: self.write(data, sections) if not self.headers_sent: # Send headers if the body was empty self.send_headers('', sections) if self.chunked and self.request_method != 'HEAD': # If chunked, send our final chunk length self.conn.sendall(b('0\r\n\r\n')) # Don't capture exceptions here. The Worker class handles # them appropriately. finally: if __debug__: self.err_log.debug('Finally closing output and sock_file') if hasattr(output, 'close'): output.close() sock_file.close()
def __init__(self, status, content_type): self.__status = status self.__ct = content_type self.__header = Headers()
def test_wrong_token(static_auth): request = mock.Mock() request.headers = Headers([("authorization", "Bearer wrong")]) assert not static_auth.is_authorized(request)
class Route(): def __init__(self, path, method, callback, status=200, content_type=None): self.path = path self.method = method self.callback = callback self.__status = status if content_type is None: content_type = 'text/html; charset=UTF-8' self.__ct = content_type self.__header = Headers() @property def status_code(self): return '{} {}'.format(self.__status, responses[self.__status]) @property def headers(self): self.__header.add_header('Strict-Transport-Security', 'max-age=31536000; includeSubDomains') self.__header.add_header('Content-Security-Policy', "default-src 'self'") self.__header.add_header('X-Content-Type-Options', 'nosniff') self.__header.add_header('X-Frame-Options', 'SAMEORIGIN') self.__header.add_header('X-XSS-Protection', '1; mode=block') self.__header.add_header('Content-type', self.__ct) return self.__header.items()
def getTileResponse(self, coord, extension, ignore_cached=False): """ Get status code, headers, and a tile binary for a given request layer tile. Arguments: - coord: one ModestMaps.Core.Coordinate corresponding to a single tile. - extension: filename extension to choose response type, e.g. "png" or "jpg". - ignore_cached: always re-render the tile, whether it's in the cache or not. This is the main entry point, after site configuration has been loaded and individual tiles need to be rendered. """ start_time = time() mimetype, format = self.getTypeByExtension(extension) # default response values status_code = 200 headers = Headers([('Content-Type', mimetype)]) body = None cache = self.config.cache if not ignore_cached: # Start by checking for a tile in the cache. try: body = cache.read(self, coord, format) except TheTileLeftANote as e: headers = e.headers status_code = e.status_code body = e.content if e.emit_content_type: headers.setdefault('Content-Type', mimetype) tile_from = 'cache' else: # Then look in the bag of recent tiles. body = _getRecentTile(self, coord, format) tile_from = 'recent tiles' # If no tile was found, dig deeper if body is None: try: lockCoord = None if self.write_cache: # this is the coordinate that actually gets locked. lockCoord = self.metatile.firstCoord(coord) # We may need to write a new tile, so acquire a lock. cache.lock(self, lockCoord, format) if not ignore_cached: # There's a chance that some other process has # written the tile while the lock was being acquired. body = cache.read(self, coord, format) tile_from = 'cache after all' if body is None: # No one else wrote the tile, do it here. buff = BytesIO() try: tile = self.render(coord, format) save = True except NoTileLeftBehind as e: tile = e.tile save = False status_code = 404 if not self.write_cache: save = False if format.lower() == 'jpeg': save_kwargs = self.jpeg_options elif format.lower() == 'png': save_kwargs = self.png_options else: save_kwargs = {} tile.save(buff, format, **save_kwargs) body = buff.getvalue() if save: cache.save(body, self, coord, format) tile_from = 'layer.render()' except TheTileLeftANote as e: headers = e.headers status_code = e.status_code body = e.content if e.emit_content_type: headers.setdefault('Content-Type', mimetype) finally: if lockCoord: # Always clean up a lock when it's no longer being used. cache.unlock(self, lockCoord, format) _addRecentTile(self, coord, format, body) logging.info( 'TileStache.Core.Layer.getTileResponse() %s/%d/%d/%d.%s via %s in %.3f', self.name(), coord.zoom, coord.column, coord.row, extension, tile_from, time() - start_time) return status_code, headers, body
class Handler(object): def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None, postexec=None): self._env = wsgienv self._start = start_resp self._meth = wsgienv.get('REQUEST_METHOD', 'GET') self._hdr = Headers([]) self._code = 0 self._msg = "unknown status" self._auth = auth self._archdir = archdir self._postexec = postexec self._loaders = loaders def send_error(self, code, message): status = "{0} {1}".format(str(code), message) self._start(status, [], sys.exc_info()) return [] def add_header(self, name, value): self._hdr.add_header(name, value) def set_response(self, code, message): self._code = code self._msg = message def end_headers(self): status = "{0} {1}".format(str(self._code), self._msg) self._start(status, list(self._hdr.items())) def handle(self): meth_handler = 'do_'+self._meth path = self._env.get('PATH_INFO', '/')[1:] if not self.authorize(): return self.send_unauthorized() if hasattr(self, meth_handler): return getattr(self, meth_handler)(path) else: return self.send_error(403, self._meth + " not supported on this resource") def authorize(self): if self._auth[0] == 'header': return self.authorize_via_headertoken() else: return self.authorize_via_queryparam() def authorize_via_queryparam(self): params = parse_qs(self._env.get('QUERY_STRING', '')) auths = params.get('auth',[]) if self._auth[1]: # match the last value provided return len(auths) > 0 and self._auth[1] == auths[-1] if len(auths) > 0: log.warning("Authorization key provided, but none has been configured") return len(auths) == 0 def authorize_via_headertoken(self): authhdr = self._env.get('HTTP_AUTHORIZATION', "") log.debug("Request HTTP_AUTHORIZATION: %s", authhdr) parts = authhdr.split() if self._auth[1]: return len(parts) > 1 and parts[0] == "Bearer" and \ self._auth[1] == parts[1] if authhdr: log.warning("Authorization key provided, but none has been configured") return authhdr == "" def send_unauthorized(self): self.set_response(401, "Not authorized") if self._auth[0] == 'header': self.add_header('WWW-Authenticate', 'Bearer') self.end_headers() return [] def do_GET(self, path): path = path.strip('/') if not path: try: out = json.dumps(list(self._loaders.keys())) + '\n' out = out.encode() except Exception as ex: log.exception("Internal error: "+str(ex)) return self.send_error(500, "Internal error") self.set_response(200, "Supported Record Types") self.add_header('Content-Type', 'application/json') self.add_header('Content-Length', str(len(out))) self.end_headers() return [out] elif path in self._loaders: self.set_response(200, "Service is ready") self.add_header('Content-Type', 'application/json') self.end_headers() return [b"Service ready\n"] else: return self.send_error(404, "resource does not exist") def do_POST(self, path): path = path.strip('/') steps = path.split('/') if len(steps) == 0: return self.send_error(405, "POST not supported on this resource") elif len(steps) == 1: if steps[0] == 'nerdm': return self.ingest_nerdm_record() else: return self.send_error(403, "new records are not allowed for " + "submission to this resource") else: return self.send_error(404, "resource does not exist") def nerdm_archive_cache(self, rec): """ cache a NERDm record into a local disk archive. The cache is for records that have been accepted but not ingested. """ try: arkid = re.sub(r'/.*$', '', re.sub(r'ark:/\d+/', '', rec['@id'])) ver = rec.get('version', '1.0.0').replace('.', '_') recid = "%s-v%s" % (os.path.basename(arkid), ver) outfile = os.path.join(self._archdir, '_cache', recid+".json") with open(outfile, 'w') as fd: json.dump(rec, fd, indent=2) return recid except KeyError as ex: # this shouldn't happen if the record was already validated raise RecordIngestError("submitted record is missing the @id "+ "property") except ValueError as ex: # this shouldn't happen if the record was already validated raise RecordIngestError("submitted record is apparently invalid; "+ "unable to submit") except OSError as ex: raise RuntimeError("Failed to cache record ({0}): {1}" .format(arkid, str(ex))) def nerdm_archive_commit(self, recid): """ commit a previously cached record to the local disk archive. This method is called after the record has been successfully ingested to the RMM's database. """ outfile = os.path.join(self._archdir, '_cache', recid+".json") if not os.path.exists(outfile): raise RuntimeError("record to commit ({0}) not found in cache: {1}" .format(recid, outfile)) try: os.rename(outfile, os.path.join(self._archdir, os.path.basename(outfile))) except OSError as ex: raise RuntimeError("Failed to archvie record ({0}): {1}" .format(recid, str(ex))) def ingest_nerdm_record(self): """ Accept a NERDm record for ingest into the RMM """ loader = self._loaders['nerdm'] try: clen = int(self._env['CONTENT_LENGTH']) except KeyError as ex: log.exception("Content-Length not provided for input record") return self.send_error(411, "Content-Length is required") except ValueError as ex: log.exception("Failed to parse input JSON record: "+str(e)) return self.send_error(400, "Content-Length is not an integer") try: bodyin = self._env['wsgi.input'] doc = bodyin.read(clen) rec = json.loads(doc) except Exception as ex: log.exception("Failed to parse input JSON record: "+str(ex)) log.warning("Input document starts...\n{0}...\n...{1} ({2}/{3} chars)" .format(doc[:75], doc[-20:], len(doc), clen)) return self.send_error(400, "Failed to load input record (bad format?): "+ str(ex)) try: recid = self.nerdm_archive_cache(rec) res = loader.load(rec, validate=True) if res.failure_count > 0: res = res.failures()[0] logmsg = "Failed to load record with "+str(res.key) for e in res.errs: logmsg += "\n "+str(e) log.error(logmsg) self.set_response(400, "Input record is not valid") self.add_header('Content-Type', 'application/json') self.end_headers() out = json.dumps([str(e) for e in res.errs]) + '\n' return [ out.encode() ] except RecordIngestError as ex: log.exception("Failed to load posted record: "+str(ex)) self.set_response(400, "Input record is not valid (missing @id)") self.add_header('Content-Type', 'application/json') self.end_headers() out = json.dumps([ "Record is missing @id property" ]) + '\n' return [ out.encode() ] except Exception as ex: log.exception("Loading error: "+str(ex)) return self.send_error(500, "Load failure due to internal error") try: self.nerdm_archive_commit(recid) except Exception as ex: log.exception("Commit error: "+str(ex)) if self._postexec: # run post-commit script try: self.nerdm_post_commit(recid) except Exception as ex: log.exception("Post-commit error: "+str(ex)) log.info("Accepted record %s with @id=%s", rec.get('ediid','?'), rec.get('@id','?')) self.set_response(200, "Record accepted") self.end_headers() return [] def nerdm_post_commit(self, recid): """ run an external executable for further processing after the record is commited to the database (e.g. update an external index) """ cmd = _mkpostcomm(self._postexec, recid) try: log.debug("Executing post-commit script:\n %s", " ".join(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = p.communicate() if p.returncode != 0: log.error("Error occurred while running post-commit script:\n"+(err or out)) except OSError as ex: log.error("Failed to execute post-commit script:\n %s\n%s", " ".join(cmd), str(ex)) except Exception as ex: log.error("Unexpected failure executing post-commit script:\n %s\n%s", " ".join(cmd), str(ex))
class Url(Packer, object): def __init__(self, id, url, cookie='', headers=HEADERS_CHROME, host=None, port=None, path=None, protocol=None, proxy=None, max_thread=-1, range_format='Range: bytes=%d-%d'): self.id = id self.url = url self.host = host if host is not None else getattr(self, 'host', None) self.port = port if port is not None else getattr(self, 'port', None) self.path = path if path is not None else getattr(self, 'path', None) self.protocol = protocol if protocol is not None else getattr( self, 'protocol', None) self.cookie = cookie if isinstance(headers, Headers): self.headers = headers elif isinstance(headers, dict): self.headers = Headers(list(headers.items())) else: raise ValueError('headers must be an instance of dict or Headers') self.etag = None self.proxy = proxy self.target = Target() self.max_thread = max_thread self.range_format = range_format def __eq__(self, other): if isinstance(other, Url): return self.url == other.url and \ self.cookie == other.cookie and \ self.proxy == other.proxy and \ self.range_format == other.range_format else: object.__eq__(self, other) def config(self): pass def getContentSize(self): if self.target.code == 200 and int( self.target.headers.get('Content-Length', -1)) != -1: return int(self.target.headers.get('Content-Length')) elif self.target.code == 206 and self.target.headers.get( 'Content-Range'): return int(self.target.headers.get('Content-Range').split('/')[-1]) else: return -1 def getFileName(self): ctd = self.target.headers.get('Content-Disposition') if ctd is not None: filename = re.findall(r'filename="(.*?)"', ctd) if filename: return filename[0] filename = self.path.split('?')[0].split('/')[-1] if filename != '': if '.' not in filename or filename.split('.')[-1] == '': extension = _content_type( self.target.headers.get('Content-Type')) filename = filename + extension else: filename = None return filename def reload(self): self.target.load(self.url) def __setattr__(self, key, value): object.__setattr__(self, key, value) if key == 'url': self.protocol, s1 = splittype(self.url) if s1: s2, self.path = splithost(s1) if s2: self.host, port = splitport(s2) self.port = int(port) if port is not None else None if not getattr(self, 'port', None): if self.protocol == 'http': self.port = 80 elif self.protocol == 'https': self.port = 443 def activate(self): res, cookie_dict = self.__request__() # if res.getcode() == 200 or res.getcode() == 206: headers_items = () if sys.version_info < (3, 0): headers_items = res.info().items() if sys.version_info >= (3, 0): headers_items = res.getheaders() self.target.update(res.geturl(), headers_items, res.getcode()) # else: # raise Exception('UrlNoRespond or UrlError') def __request__(self): Cookiejar = CookieJar() opener = build_opener(HTTPCookieProcessor(Cookiejar)) _header = dict(self.headers.items()) if self.cookie: _header.update({'Cookie': self.cookie}) req = Request(self.url, headers=_header, origin_req_host=self.host) error_counter = 0 while error_counter < 3: try: res = opener.open(req) break except Exception as e: # traceback.print_exc() error_counter += 1 time.sleep(0.5) else: raise Exception('UrlNotRespond') return res, Cookiejar._cookies def getHeader(self, name, default=None): return self.headers.get(name, default) def __packet_params__(self): return [ 'id', 'url', 'host', 'port', 'protocal', 'cookie', 'etag', 'proxy', 'max_thread', 'range_format', 'headers' ]
class MultipartPart(object): def __init__(self, buffer_size=2 ** 16, memfile_limit=2 ** 18, charset='latin1'): self.headerlist = [] self.headers = None self.file = False self.size = 0 self._buf = tob('') self.disposition, self.name, self.filename = None, None, None self.content_type, self.charset = None, charset self.memfile_limit = memfile_limit self.buffer_size = buffer_size def feed(self, line, nl=''): if self.file: return self.write_body(line, nl) return self.write_header(line, nl) def write_header(self, line, nl): line = line.decode(self.charset or 'latin1') if not nl: raise MultipartError('Unexpected end of line in header.') if not line.strip(): # blank line -> end of header segment self.finish_header() elif line[0] in ' \t' and self.headerlist: name, value = self.headerlist.pop() self.headerlist.append((name, value + line.strip())) else: if ':' not in line: raise MultipartError("Syntax error in header: No colon.") name, value = line.split(':', 1) self.headerlist.append((name.strip(), value.strip())) def write_body(self, line, nl): if not line and not nl: return # This does not even flush the buffer self.size += len(line) + len(self._buf) self.file.write(self._buf + line) self._buf = nl if self.content_length > 0 and self.size > self.content_length: raise MultipartError('Size of body exceeds Content-Length header.') if self.size > self.memfile_limit and isinstance(self.file, BytesIO): # TODO: What about non-file uploads that exceed the memfile_limit? self.file, old = TemporaryFile(mode='w+b'), self.file old.seek(0) copy_file(old, self.file, self.size, self.buffer_size) def finish_header(self): self.file = BytesIO() self.headers = Headers(self.headerlist) cdis = self.headers.get('Content-Disposition', '') ctype = self.headers.get('Content-Type', '') self.headers.get('Content-Length', '-1') if not cdis: raise MultipartError('Content-Disposition header is missing.') self.disposition, self.options = parse_options_header(cdis) self.name = self.options.get('name') self.filename = self.options.get('filename') self.content_type, options = parse_options_header(ctype) self.charset = options.get('charset') or self.charset self.content_length = int(self.headers.get('Content-Length', '-1')) def is_buffered(self): ''' Return true if the data is fully buffered in memory.''' return isinstance(self.file, BytesIO) @property def value(self): ''' Data decoded with the specified charset ''' pos = self.file.tell() self.file.seek(0) val = self.file.read() self.file.seek(pos) return val.decode(self.charset) def save_as(self, path): fp = open(path, 'wb') pos = self.file.tell() try: self.file.seek(0) size = copy_file(self.file, fp) finally: self.file.seek(pos) return size
def _handle_http_response_start(self, message: Dict[str, Any]): self._headers = Headers( [(k.decode(), v.decode()) for k, v in message["headers"]]) self._status_code = message["status"]