def static_file_view(env, start_response, filename, block_size, charset, CACHE_DURATION):
method = env['REQUEST_METHOD'].upper()
if method not in ('HEAD', 'GET'):
start_response('405 METHOD NOT ALLOWED',
[('Content-Type', 'text/plain; UTF-8')])
return [b'']
mimetype, encoding = mimetypes.guess_type(filename)
headers = Headers([])
cache_days = CACHE_DURATION.get(mimetype, 0)
expires = datetime.datetime.utcnow() + datetime.timedelta(cache_days)
headers.add_header('Cache-control', f'public, max-age={expires.strftime(RFC_1123_DATE)}')
headers.add_header('Expires', expires.strftime(RFC_1123_DATE))
if env.get('HTTP_IF_MODIFIED_SINCE'):
if env.get('HTTP_IF_MODIFIED_SINCE') >= generate_last_modified(filename):
start_response('304 ok', headers.items())
return [b'304']
headers.add_header('Content-Encodings', encoding)
if mimetype:
headers.add_header('Content-Type', get_content_type(mimetype, charset))
headers.add_header('Content-Length', get_content_length(filename))
headers.add_header('Last-Modified', generate_last_modified(filename))
headers.add_header("Accept-Ranges", "bytes")
start_response('200 OK', headers.items())
return _get_body(filename, method, block_size, charset)
def handle(environ, start_response):
with self.db_env.begin(self.metadata_db, write=True) as tx:
message = pickle.loads(tx.get(message_id.encode('ascii')))
self._update_tracking(
tx,
message_id,
message,
downloadTimestamp=datetime.datetime.now().strftime(
TIMESTAMP_FORMAT))
assert message.recipient == mailbox
status = '206 Partial Content' if message.chunks > chunk_num else '200 OK'
chunk_header = "{}:{}".format(chunk_num, message.chunks)
headers = Headers([('Content-Type', 'application/octet-stream'),
('Mex-Chunk-Range', chunk_header),
('Mex-MessageID', str(message_id))])
for k, v in message.extra_headers.items():
headers[k] = v
f = open(self.get_filename(mailbox, message_id, chunk_num), 'rb')
if "gzip" in environ.get('HTTP_ACCEPT_ENCODING', ''):
headers['Content-Encoding'] = 'gzip'
start_response(status, headers.items())
return wrap_file(environ, f)
else:
start_response(status, headers.items())
return decompress_file(f)
class Response:
def __init__(self):
self.status = '200 OK'
self.headers = Headers([('Content-Type', 'text/html; charset=utf-8')])
def send(self, start_response):
start_response(self.status, self.headers.items())
def board_app(env, resp):
path = env['PATH_INFO']
m = board_re.match(path)
board = m.group(1)
message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja'))
headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')])
resp("200 OK", headers.items())
board = utils.sanitize(utils.get_board(path))
if board:
fmt = '{logo} - {board} - {desc}'
else:
fmt = '{logo} - {desc}'
text = fmt.format(logo=message['logo'], desc=message['description'], board=board)
html = '''
<!DOCTYPE html>
<html><head>
<meta http-equiv="content-type" content="text/html; charset=Shift_JIS">
<title>{text}</title>
<meta name="description" content="{text}">
</head><body>
<h1>{text}</h1>
</body></html>
'''.format(text=text)
return [html.encode('cp932', 'replace')]
def thread_app(env, resp):
path = env['PATH_INFO']
# utils.log('thread_app', path)
m = thread_re.match(path)
board, datkey = m.group(1), m.group(2)
key = keylib.get_filekey(datkey)
data = cache.Cache(key)
data.load()
if check_get_cache(env):
if not data.exists() or len(data) == 0:
# when first access, load data from network
data.search()
elif _count_is_update(key):
# update thread
# limit `data.search` calling. it's slow!
threading.Thread(target=data.search, daemon=True).start()
if not data.exists():
resp('404 Not Found', [('Content-Type', 'text/plain; charset=Shift_JIS')])
return [b'404 Not Found']
thread = dat.make_dat(data, env, board)
headers = Headers([('Content-Type', 'text/plain; charset=Shift_JIS')])
last_m = eutils.formatdate(data.stamp)
headers['Last-Modified'] = last_m
resp("200 OK", headers.items())
return (c.encode('cp932', 'replace') for c in thread)
class Response:
default_status = '200 OK'
default_content_type = 'text/html; charset=UTF-8'
def __init__(self, body='', status=None, headers=None, charset='utf-8'):
self._body = body
self.status = status or self.default_status
self.headers = Headers()
self.charset = charset
if headers:
for name, value in headers.items():
self.headers.add_header(name, value)
@property
def body(self):
if isinstance(self._body, str):
return self._body.encode(self.charset)
return self._body
@property
def header_list(self):
if 'Content-Type' not in self.headers:
self.headers.add_header('Content-Type', self.default_content_type)
return self.headers.items()
def handle_application_error(self, environ, start_response):
status = "500 Internal Server Error"
headers = Headers([])
# Package the exception info as into a special header and
# send it to the client
type, exc, tb = sys.exc_info()
tbfile = StringIO()
traceback.print_exc(file=tbfile)
headers['Content-Type'] = 'text/plain; charset=utf-8'
LOG.debug("Packing traceback context into debug header: %s",
self.debug_header)
debug_header = self.pack_header(Traceback(tb))
LOG.debug("Debug header (%d bytes): %s",
len(debug_header), debug_header)
headers[self.debug_header] = debug_header
app_uri = application_uri(environ)
headers["Location"] = app_uri[:-1] + self.debug_uri
start_response(status, headers.items())
return [tbfile.getvalue().encode('utf-8')]
class Response:
def __init__(self,
response=None,
status=200,
charset="utf-8",
content_type="text/html"):
self._headers = Headers()
self.response = [] if response is None else response
self.charset = charset
self.content_type = f"{content_type}; {charset}"
self._headers.add_header("content_type", content_type)
self._status = status
@property
def status(self):
status_string = http.client.responses.get(self._status, 'UNKNOWN')
return '{status} {status_string}'.format(status=self._status,
status_string=status_string)
@property
def headers(self):
return self._headers.items()
def __iter__(self):
for v in self.response:
if isinstance(v, bytes):
yield v
else:
yield v.encode(self.charset)
class Response:
default_status = 200
default_charset = "utf-8"
default_content_type = "text/html; charset=UTF-8"
def __init__(self, body="", status=None, headers=None, charset=None):
self._body = body
self.status = status or self.default_status
self.headers = Headers()
self.charset = charset or self.default_charset
if headers:
for name, value in headers.items():
self.headers.add_header(name, value)
@property
def status_code(self):
return "%d %s" % (self.status, http_responses[self.status])
@property
def header_list(self):
if "Content-Type" not in self.headers:
self.headers.add_header("Content-Type", self.default_content_type)
return self.headers.items()
@property
def body(self):
if isinstance(self._body, str):
return [self._body.encode(self.charset)]
return [self._body]
class Route():
def __init__(self, path, method, callback, status=200, content_type=None):
self.path = path
self.method = method
self.callback = callback
self.__status = status
if content_type is None: content_type = 'text/html; charset=UTF-8'
self.__ct = content_type
self.__header = Headers()
@property
def status_code(self):
return '{} {}'.format(self.__status, responses[self.__status])
@property
def headers(self):
self.__header.add_header('Strict-Transport-Security', 'max-age=31536000; includeSubDomains')
self.__header.add_header('Content-Security-Policy', "default-src 'self'")
self.__header.add_header('X-Content-Type-Options', 'nosniff')
self.__header.add_header('X-Frame-Options', 'SAMEORIGIN')
self.__header.add_header('X-XSS-Protection', '1; mode=block')
self.__header.add_header('Content-type', self.__ct)
return self.__header.items()
class Response:
id: str
headers: Headers
request: Request
status_code: int
body: bytes
def __init__(self: 'Response', start_response, request: Request):
self.request = request
self.id = request.id
self._start_response = start_response
self.headers = Headers()
self.status_code = 200
def status(self: 'Response', status_code: int) -> 'Response':
self.status_code = status_code
return self
def json(self: 'Response', data: Dict):
self.headers.add_header(CONTENT_TYPE, 'application/json')
return self.send(json.dumps(data).encode(UTF_8))
def send(self: 'Response', body: bytes):
self.body = body
self.headers.add_header(CONTENT_LENGTH, str(len(self.body)))
self._start_response(self._get_wsgi_http_status(self.status_code),
self.headers.items())
return self.body
@staticmethod
def _get_wsgi_http_status(status_code: int) -> str:
for http_status in HTTPStatus:
if http_status.value == status_code:
return '%d %s' % (http_status.value, http_status.phrase)
raise ValueError("Unsupported HTTP Code: '%d'" % status_code)
def newapp(environ, start_response):
body = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
already = 'Content-Encoding' in headers
accepted = 'gzip' in environ.get('HTTP_ACCEPT_ENCODING', '')
if not accepted or already:
# no compress
start_response(status, list(headers.items()))
return body
content = gzip.compress(b''.join(body))
if hasattr(body, 'close'):
body.close()
headers['Content-Encoding'] = 'gzip'
start_response(status, list(headers.items()))
return [content]
def newapp(environ, start_response):
body = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
already = 'Content-Encoding' in headers
accepted = 'gzip' in environ.get('HTTP_ACCEPT_ENCODING', '')
if not accepted or already:
# no compress
start_response(status, list(headers.items()))
return body
content = gzip.compress(b''.join(body))
if hasattr(body, 'close'):
body.close()
headers['Content-Encoding'] = 'gzip'
start_response(status, list(headers.items()))
return [content]
def newapp(environ, start_response):
raw = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
if (not 'Last-Modified' in headers
or not environ.get('HTTP_IF_MODIFIED_SINCE')):
start_response(status, list(headers.items()))
return raw
last_m = eutils.parsedate(headers['Last-Modified'])
since_m = eutils.parsedate(environ['HTTP_IF_MODIFIED_SINCE'])
if since_m < last_m:
start_response(status, list(headers.items()))
return raw
else:
start_response('304 Not Modified', list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [b'']
def newapp(environ, start_response):
raw = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
if (not 'Last-Modified' in headers
or not environ.get('HTTP_IF_MODIFIED_SINCE')):
start_response(status, list(headers.items()))
return raw
last_m = eutils.parsedate(headers['Last-Modified'])
since_m = eutils.parsedate(environ['HTTP_IF_MODIFIED_SINCE'])
if since_m < last_m:
start_response(status, list(headers.items()))
return raw
else:
start_response('304 Not Modified', list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [b'']
def get_alternatives(base_headers, files):
# Sort by size so that the smallest compressed alternative matches first
alternatives = []
files_by_size = sorted(files.items(), key=lambda i: i[1].stat.st_size)
for encoding, file_entry in files_by_size:
headers = Headers(base_headers.items())
headers['Content-Length'] = str(file_entry.stat.st_size)
if encoding:
headers['Content-Encoding'] = encoding
encoding_re = re.compile(r'\b%s\b' % encoding)
else:
encoding_re = re.compile('')
alternatives.append((encoding_re, file_entry.path, headers.items()))
return alternatives
def get_alternatives(base_headers, files):
# Sort by size so that the smallest compressed alternative matches first
alternatives = []
files_by_size = sorted(files.items(), key=lambda i: i[1].stat.st_size)
for encoding, file_entry in files_by_size:
headers = Headers(base_headers.items())
headers["Content-Length"] = str(file_entry.stat.st_size)
if encoding:
headers["Content-Encoding"] = encoding
encoding_re = re.compile(r"\b%s\b" % encoding)
else:
encoding_re = re.compile("")
alternatives.append((encoding_re, file_entry.path, headers.items()))
return alternatives
def __init__(self, url, headers):
self._url = None
self._urlparse = None
if headers is None:
headers = Headers([])
elif isinstance(headers, dict):
headers = Headers(list(headers.items()))
elif isinstance(headers, list):
headers = Headers(headers)
else:
raise TypeError()
self.headers = headers
self.url = url
class Static(Route):
def __init__(self, status, content_type):
self.__status = status
self.__ct = content_type
self.__header = Headers()
@property
def status_code(self):
return '{} {}'.format(self.__status, responses[self.__status])
@property
def headers(self):
self.__header.add_header('Content-type', self.__ct)
return self.__header.items()
def get_static_file(self, path, url, stat_cache=None):
# Optimization: bail early if file does not exist
if stat_cache is None and not os.path.exists(path):
raise MissingFileError(path)
headers = Headers([])
self.add_mime_headers(headers, path, url)
self.add_cache_headers(headers, path, url)
if self.allow_all_origins:
headers['Access-Control-Allow-Origin'] = '*'
if self.add_headers_function:
self.add_headers_function(headers, path, url)
return StaticFile(
path, headers.items(),
stat_cache=stat_cache,
encodings={
'gzip': path + '.gz', 'br': path + '.br'})
def get_static_file(self, path, url, stat_cache=None):
# Optimization: bail early if file does not exist
if stat_cache is None and not os.path.exists(path):
raise MissingFileError(path)
headers = Headers([])
self.add_mime_headers(headers, path, url)
self.add_cache_headers(headers, path, url)
if self.allow_all_origins:
headers['Access-Control-Allow-Origin'] = '*'
if self.add_headers_function:
self.add_headers_function(headers, path, url)
return StaticFile(
path, headers.items(),
stat_cache=stat_cache,
encodings={
'gzip': path + '.gz', 'br': path + '.br'})
def __call__(self):
"""Set the headers and return the body.
It is not necessary to supply Content-length, because this is added by
the caller.
"""
headers = Headers([])
content_type_params = {}
if self.charset is not None:
content_type_params['charset'] = self.charset
headers.add_header(
'Content-Type', self.content_type, **content_type_params)
headers.add_header(
'Content-Disposition', 'attachment', filename=self.filename)
for key, value in headers.items():
self.request.response.setHeader(key, value)
return self.getBody()
def static_file_view(env, start_response, filename, block_size, charset):
method = env['REQUEST_METHOD'].upper()
if method not in ('HEAD', 'GET'):
start_response('405 METHOD NOT ALLOWED',
[('Content-Type', 'text/plain; UTF-8')])
return [b'']
mimetype, encoding = mimetypes.guess_type(filename)
headers = Headers([])
headers.add_header('Content-Encodings', encoding)
headers.add_header('Content-Type', get_content_type(mimetype, charset))
headers.add_header('Content-Length', get_content_length(filename))
headers.add_header('Last-Modified', generate_last_modified())
headers.add_header("Accept-Ranges", "bytes")
start_response('200 OK', headers.items())
return _get_body(filename, method, block_size, charset)
class Response(threading.local):
""" Represents a single response using thread-local namespace. """
def bind(self):
""" Clears old data and creates a brand new Response object """
self._COOKIES = None
self.status = 200
self.header_list = []
self.header = HeaderWrapper(self.header_list)
self.content_type = 'text/html'
self.error = None
self.charset = 'utf8'
def wsgiheaders(self):
''' Returns a wsgi conform list of header/value pairs '''
for c in self.COOKIES.itervalues():
self.header.add_header('Set-Cookie', c.OutputString())
return [(h.title(), str(v)) for h, v in self.header.items()]
@property
def COOKIES(self):
if not self._COOKIES:
self._COOKIES = SimpleCookie()
return self._COOKIES
def set_cookie(self, key, value, **kargs):
"""
Sets a Cookie. Optional settings:
expires, path, comment, domain, max-age, secure, version, httponly
"""
self.COOKIES[key] = value
for k, v in kargs.iteritems():
self.COOKIES[key][k] = v
def get_content_type(self):
""" Get the current 'Content-Type' header. """
return self.header['Content-Type']
def set_content_type(self, value):
if 'charset=' in value:
self.charset = value.split('charset=')[-1].split(';')[0].strip()
self.header['Content-Type'] = value
content_type = property(get_content_type, set_content_type, None,
get_content_type.__doc__)
def board_app(env, resp):
path = env['PATH_INFO']
m = board_re.match(path)
board = m.group(1)
message = gateway.search_message(env.get('HTTP_ACCEPT_LANGUAGE', 'ja'))
headers = Headers([('Content-Type', 'text/html; charset=Shift_JIS')])
resp("200 OK", headers.items())
html = [
'<!DOCTYPE html>',
'<html><head>',
'<meta http-equiv="content-type" content="text/html; charset=Shift_JIS">',
'<title>%s - %s</title>' % (message['logo'], message['description']),
'<meta name="description" content="%s - %s">' % (message['logo'], message['description']),
'</head><body>',
'<h1>%s - %s</h1>' % (message['logo'], message['description']),
'</body></html>',
]
return ((c + '\n').encode('sjis', 'ignore') for c in html)
def do_GET(self, environ, start_response):
headers = Headers()
video_url = "{PATH_INFO}?{QUERY_STRING}".format(**environ).strip("/")
if video_url == "?":
raise self.HTTPError(400, more="no URL provided")
try:
videos = self.downloader.get_videos(video_url)
except CannotDownload as cad:
raise self.HTTPError(400, "Cannot download", more=str(cad))
if len(videos) != 1:
raise self.HTTPError(400, more="playlists not supported yet")
video = videos[0]
audio_file = self.downloader.cache_dir / video.path
assert audio_file.exists()
filesize = audio_file.stat().st_size
headers.add_header("Content-Disposition",
"attachment",
filename=video.title)
headers.add_header("Content-Type", "audio/mpeg")
headers.add_header("Content-Length", str(filesize))
start_response("200 OK", headers.items())
return FileWrapper(audio_file.open("rb"))
class BaseResponse:
"""Base class for Response"""
default_status = 200
default_content_type = 'text/plain;'
def __init__(self, body=b'', status=None, headers=None):
self.headers = Headers()
self._body = body
self._status_code = status or self.default_status
self._cookies = SimpleCookie()
if headers:
for name, value in headers.items():
self.headers.add_header(name, value)
@property
def body(self):
return [self._body]
@property
def status_code(self):
""" The HTTP status code as an integer (e.g. 404)."""
return self._status_code
@property
def status(self):
""" The HTTP status line as a string (e.g. ``404 Not Found``)."""
if not 100 <= self._status_code <= 999:
raise ValueError('Status code out of range.')
status = _HTTP_STATUS_LINES.get(self._status_code)
return str(status or ('{} Unknown'.format(self._status_code)))
@status.setter
def status(self, status_code):
if not 100 <= status_code <= 999:
raise ValueError('Status code out of range.')
self._status_code = status_code
@property
def headerlist(self):
""" WSGI conform list of (header, value) tuples. """
if 'Content-Type' not in self.headers:
self.headers.add_header('Content-Type', self.default_content_type)
if self._cookies:
for c in self._cookies.values():
self.headers.add_header('Set-Cookie', c.OutputString())
return self.headers.items()
def set_cookie(self, key, value, expires=None, max_age=None, path=None,
secret=None, digestmod=hashlib.sha256):
if secret:
if isinstance(secret, str):
secret = secret.encode('utf-8')
encoded = base64.b64encode(pickle.dumps((key, value), pickle.HIGHEST_PROTOCOL))
sig = base64.b64encode(hmac.new(secret, encoded, digestmod=digestmod).digest())
value_bytes = b'!' + sig + b'?' + encoded
value = value_bytes.decode('utf-8')
self._cookies[key] = value
if len(key) + len(value) > 3800:
raise ValueError('Content does not fit into a cookie.')
if max_age is not None:
if isinstance(max_age, int):
max_age_value = max_age
else:
max_age_value = max_age.seconds + max_age.days * 24 * 3600
self._cookies[key]['max-age'] = max_age_value
if expires is not None:
if isinstance(expires, int):
expires_value = expires
else:
expires_value = time.strftime("%a, %d %b %Y %H:%M:%S GMT", expires.timetuple())
self._cookies[key]['expires'] = expires_value
if path:
self._cookies[key]['path'] = path
def delete_cookie(self, key, **kwargs):
kwargs['max_age'] = -1
kwargs['expires'] = 0
self.set_cookie(key, '', **kwargs)
def newapp(environ, start_response):
raw = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
headers.setdefault('Accept-Range', 'bytes')
range = environ.get('HTTP_RANGE')
if (range is None or ',' in range # not deal with multi-part range
or not status.startswith('2')): # not success status
start_response(status, list(headers.items()))
return raw
def error_416():
start_response('416 Requested Range Not Satisfiable',
list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [b'']
m = re.match(r'bytes=([0-9]+)?-([0-9]+)?', range)
if not m or (not m.group(1) and not m.group(2)):
return error_416()
content = b''.join(raw)
begin = int(m.group(1)) if m.group(1) else None
end = int(m.group(2)) if m.group(2) else None
# because 0 is False
has_begin = begin is not None
has_end = end is not None
if (has_begin and has_end) and end < begin:
return error_416()
if has_end and len(content) <= end:
return error_416()
if has_begin and len(content) <= begin:
return error_416()
if has_begin and has_end:
# bytes=begin-end
c_range = 'bytes {}-{}/{}'.format(begin, end, len(content))
body = content[begin:end + 1]
elif has_begin:
# bytes=begin-
c_range = 'bytes {}-{}/{}'.format(begin,
len(content) - 1, len(content))
body = content[begin:]
else:
# bytes=-end
c_range = 'bytes {}-{}/{}'.format(
len(content) - end,
len(content) - 1, len(content))
body = content[len(content) - end:]
headers['Content-Range'] = c_range
start_response('206 Partial Content', list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [body]
class Url(Packer, object):
def __init__(self,
id,
url,
cookie='',
headers=HEADERS_CHROME,
host=None,
port=None,
path=None,
protocol=None,
proxy=None,
max_thread=-1,
range_format='Range: bytes=%d-%d'):
self.id = id
self.url = url
self.host = host if host is not None else getattr(self, 'host', None)
self.port = port if port is not None else getattr(self, 'port', None)
self.path = path if path is not None else getattr(self, 'path', None)
self.protocol = protocol if protocol is not None else getattr(
self, 'protocol', None)
self.cookie = cookie
if isinstance(headers, Headers):
self.headers = headers
elif isinstance(headers, dict):
self.headers = Headers(list(headers.items()))
else:
raise ValueError('headers must be an instance of dict or Headers')
self.etag = None
self.proxy = proxy
self.target = Target()
self.max_thread = max_thread
self.range_format = range_format
def __eq__(self, other):
if isinstance(other, Url):
return self.url == other.url and \
self.cookie == other.cookie and \
self.proxy == other.proxy and \
self.range_format == other.range_format
else:
object.__eq__(self, other)
def config(self):
pass
def getContentSize(self):
if self.target.code == 200 and int(
self.target.headers.get('Content-Length', -1)) != -1:
return int(self.target.headers.get('Content-Length'))
elif self.target.code == 206 and self.target.headers.get(
'Content-Range'):
return int(self.target.headers.get('Content-Range').split('/')[-1])
else:
return -1
def getFileName(self):
ctd = self.target.headers.get('Content-Disposition')
if ctd is not None:
filename = re.findall(r'filename="(.*?)"', ctd)
if filename:
return filename[0]
filename = self.path.split('?')[0].split('/')[-1]
if filename != '':
if '.' not in filename or filename.split('.')[-1] == '':
extension = _content_type(
self.target.headers.get('Content-Type'))
filename = filename + extension
else:
filename = None
return filename
def reload(self):
self.target.load(self.url)
def __setattr__(self, key, value):
object.__setattr__(self, key, value)
if key == 'url':
self.protocol, s1 = splittype(self.url)
if s1:
s2, self.path = splithost(s1)
if s2:
self.host, port = splitport(s2)
self.port = int(port) if port is not None else None
if not getattr(self, 'port', None):
if self.protocol == 'http':
self.port = 80
elif self.protocol == 'https':
self.port = 443
def activate(self):
res, cookie_dict = self.__request__()
# if res.getcode() == 200 or res.getcode() == 206:
headers_items = ()
if sys.version_info < (3, 0):
headers_items = res.info().items()
if sys.version_info >= (3, 0):
headers_items = res.getheaders()
self.target.update(res.geturl(), headers_items, res.getcode())
# else:
# raise Exception('UrlNoRespond or UrlError')
def __request__(self):
Cookiejar = CookieJar()
opener = build_opener(HTTPCookieProcessor(Cookiejar))
_header = dict(self.headers.items())
if self.cookie:
_header.update({'Cookie': self.cookie})
req = Request(self.url, headers=_header, origin_req_host=self.host)
error_counter = 0
while error_counter < 3:
try:
res = opener.open(req)
break
except Exception as e:
# traceback.print_exc()
error_counter += 1
time.sleep(0.5)
else:
raise Exception('UrlNotRespond')
return res, Cookiejar._cookies
def getHeader(self, name, default=None):
return self.headers.get(name, default)
def __packet_params__(self):
return [
'id', 'url', 'host', 'port', 'protocal', 'cookie', 'etag', 'proxy',
'max_thread', 'range_format', 'headers'
]
class BeanServer(object):
"A really, really simple application server."
default_headers = [('Content-Type', 'text/html')]
def __init__(self, ledger, opts):
self.ledger = ledger
self.data = []
self.load()
# Map of session to dict.
self.cookiejar = {}
# Prototype for context object.
ctx = self.ctx = Context()
self.opts = ctx.opts = opts
ctx.debug = opts.debug
def setHeader(self, name, value):
self.headers[name] = value
def write(self, data):
assert isinstance(data, str), data
self.data.append(data)
def load(self):
"Load the application pages."
import app
reload(app)
self.mapper = app.mapper
def __call__(self, environ, start_response):
if self.ctx.debug:
self.load()
self.environ = environ
self.response = start_response
del self.data[:]
self.headers = Headers(self.default_headers)
ctx = copy(self.ctx) # shallow
ctx.ledger = self.ledger
path = environ['PATH_INFO']
ishtml = '.' not in basename(path) or path.endswith('.html')
if ishtml:
# Load cookie (session is only in memory).
cookie = Cookie.SimpleCookie(environ.get('HTTP_COOKIE', ''))
has_cookie = (bool(cookie) and
'session' in cookie and
cookie["session"].value in self.cookiejar)
if has_cookie:
session_id = cookie["session"].value
session = self.cookiejar[session_id]
else:
session_id = '%x' % randint(0, 16**16)
cookie["session"] = session_id
session = self.cookiejar[session_id] = {}
ctx.session = session
try:
# Linear search in the regexp to match the request path.
page, vardict = self.mapper.match(path)
if page is None:
raise HttpNotFound(path)
else:
# Update the context object with components of the request and
# with the query parameters.
ctx.environ = environ
form = cgi.parse(environ=environ)
## FIXME: make this wsgi compatible.
## conlen = int(self.environ['CONTENT_LENGTH'])
## s = self.environ['wsgi.input'].read(conlen)
## form = cgi.parse_qs(s)
ctx.__dict__.update(form)
ctx.__dict__.update(vardict)
page(self, ctx)
# Add session cookie to headers, if necessary.
if ishtml and not has_cookie:
for k, v in sorted(cookie.items()):
self.headers.add_header('Set-Cookie', v.OutputString())
start_response('200 OK', self.headers.items())
return self.data
except HttpRedirect, e:
location = e.message
start_response(e.status, [('Location', location)])
return [str(e)]
except HttpError, e:
status = getattr(e, 'status', '500 Internal Server Error')
start_response(status, [('Content-Type', 'text/html')])
return [str(e)]
class SimIngestHandler(object):
def __init__(self, wsgienv, start_resp):
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
self._auth = (authmeth, authkey)
def send_error(self, code, message):
status = "{0} {1}".format(str(code), message)
self._start(status, [], sys.exc_info())
return []
def add_header(self, name, value):
self._hdr.add_header(name, value)
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
status = "{0} {1}".format(str(self._code), self._msg)
self._start(status, self._hdr.items())
def handle(self, env, start_resp):
meth_handler = 'do_' + self._meth
path = self._env.get('PATH_INFO', '/')[1:]
params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
print("AUTH METHOD: %s" % self._auth[0], file=sys.stderr)
if not self.authorize():
return self.send_unauthorized()
if hasattr(self, meth_handler):
return getattr(self, meth_handler)(path, params)
else:
return self.send_error(
403, self._meth + " not supported on this resource")
def authorize(self):
if self._auth[0] == 'header':
return self.authorize_via_headertoken()
else:
return self.authorize_via_queryparam()
def authorize_via_queryparam(self):
params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
auths = params.get('auth', [])
if self._auth[1]:
# match the last value provided
return len(auths) > 0 and self._auth[1] == auths[-1]
if len(auths) > 0:
log.warn(
"Authorization key provided, but none has been configured")
return len(auths) == 0
def authorize_via_headertoken(self):
authhdr = self._env.get('HTTP_AUTHORIZATION', "")
print("Request HTTP_AUTHORIZATION: %s" % authhdr, file=sys.stderr)
parts = authhdr.split()
if self._auth[1]:
return len(parts) > 1 and parts[0] == "Bearer" and \
self._auth[1] == parts[1]
if authhdr:
log.warn(
"Authorization key provided, but none has been configured")
return authhdr == ""
def send_unauthorized(self):
self.set_response(401, "Not authorized")
if self._auth[0] == 'header':
self.add_header('WWW-Authenticate', 'Bearer')
self.end_headers()
return []
def do_GET(self, path, params=None):
path = path.strip('/')
if not path:
try:
out = json.dumps(["nerdm", "invalid"]) + '\n'
except Exception, ex:
return self.send_error(500, "Internal error")
self.set_response(200, "Supported Record Types")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return [out]
elif path in "nerdm invalid".split():
self.set_response(200, "Service is ready")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return ["Service ready\n"]
class Shortly(object):
def __init__(self):
self.url_map = {}
self.view_functions = {}
self.headers = Headers()
self.status = None
def add_url_rule(self, rule, endpoint=None, view_func=None, **options):
if endpoint is None:
endpoint = _endpoint_from_view_func(view_func)
options["endpoint"] = endpoint
methods = options.pop("methods", None)
# if the methods are not given and the view_func object knows its
# methods we can use that instead. If neither exists, we go with
# a tuple of only ``GET`` as default.
if methods is None:
methods = getattr(view_func, "methods", None) or ("GET", )
if isinstance(methods, str):
raise TypeError("Allowed methods have to be iterables of strings, "
'for example: @app.route(..., methods=["POST"])')
methods = set(item.upper() for item in methods)
rule = Rule(rule, methods=methods, **options)
self.url_map.add(rule)
if view_func is not None:
self.view_functions[endpoint] = view_func
def route(self, rule, **options):
def decorator(f):
endpoint = options.pop("endpoint", None)
self.add_url_rule(rule, endpoint, f, **options)
return f
return decorator
def request_context(self, environ):
"""
:param environ: a WSGI environment
"""
request = DotDict()
request.scheme = util.guess_scheme(environ)
request.uri = util.request_uri(environ)
request.address = util.application_uri(environ)
request.path = util.shift_path_info(environ)
if environ.get('REQUEST_METHOD', None):
request.method = environ['REQUEST_METHOD']
if environ.get('CONTENT_TYPE', None):
self.headers.add_header('CONTENT_TYPE', environ['CONTENT_TYPE'])
try:
length = int(environ.get('CONTENT_LENGTH', '0'))
request.body = environ['wsgi.input'].read(length)
except ValueError:
request.body = b''
return request
def dispatch_request(self, request):
if request.path.startswith('/static'):
fn = os.path.join(path, request.path[1:])
if '.' not in fn.split(os.path.sep)[-1]:
fn = os.path.join(fn, 'index.html')
type = mimetypes.guess_type(fn)[0]
if os.path.exists(fn):
self.status = '200 OK'
self.headers.add_header('Content-type', type)
return util.FileWrapper(open(fn, "rb"))
else:
self.status = '404 Not Found'
self.headers.add_header('Content-type', 'text/plain')
return [b'not found']
try:
self.status = '200 OK'
body = json.loads(request.body.decode('utf-8'))
#rule = request.url_rule
#return self.view_functions[rule.endpoint](**req.view_args)
return body
except Exception as e:
self.status = '500 server error'
return str(e)
def wsgi_app(self, environ, start_response):
ctx = self.request_context(environ)
try:
try:
response = self.dispatch_request(ctx)
headers = [(k, v) for k, v in self.headers.items()]
start_response(self.status, headers)
return response
except Exception as e:
start_response('500 server error',
[('Content-type', 'text/plain')])
return [str(e)]
finally:
pass
def __call__(self, environ, start_response):
return self.wsgi_app(environ, start_response)
def newapp(environ, start_response):
raw = app(environ, capture)
status = resp['status']
headers = Headers(resp['headers'])
headers.setdefault('Accept-Range', 'bytes')
range = environ.get('HTTP_RANGE')
if (range is None
or ',' in range # not deal with multi-part range
or not status.startswith('2')): # not success status
start_response(status, list(headers.items()))
return raw
def error_416():
start_response('416 Requested Range Not Satisfiable',
list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [b'']
m = re.match(r'bytes=([0-9]+)?-([0-9]+)?', range)
if not m or (not m.group(1) and not m.group(2)):
return error_416()
content = b''.join(raw)
begin = int(m.group(1)) if m.group(1) else None
end = int(m.group(2)) if m.group(2) else None
# because 0 is False
has_begin = begin is not None
has_end = end is not None
if (has_begin and has_end) and end < begin:
return error_416()
if has_end and len(content) <= end:
return error_416()
if has_begin and len(content) <= begin :
return error_416()
if has_begin and has_end:
# bytes=begin-end
c_range = 'bytes {}-{}/{}'.format(begin, end, len(content))
body = content[begin:end+1]
elif has_begin:
# bytes=begin-
c_range = 'bytes {}-{}/{}'.format(begin, len(content)-1,
len(content))
body = content[begin:]
else:
# bytes=-end
c_range = 'bytes {}-{}/{}'.format(len(content)-end,
len(content)-1, len(content))
body = content[len(content)-end:]
headers['Content-Range'] = c_range
start_response('206 Partial Content', list(headers.items()))
if hasattr(raw, 'close'):
raw.close()
return [body]
class Handler(object):
def __init__(self, loaders, wsgienv, start_resp, archdir, auth=None, postexec=None):
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
self._auth = auth
self._archdir = archdir
self._postexec = postexec
self._loaders = loaders
def send_error(self, code, message):
status = "{0} {1}".format(str(code), message)
self._start(status, [], sys.exc_info())
return []
def add_header(self, name, value):
self._hdr.add_header(name, value)
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
status = "{0} {1}".format(str(self._code), self._msg)
self._start(status, list(self._hdr.items()))
def handle(self):
meth_handler = 'do_'+self._meth
path = self._env.get('PATH_INFO', '/')[1:]
if not self.authorize():
return self.send_unauthorized()
if hasattr(self, meth_handler):
return getattr(self, meth_handler)(path)
else:
return self.send_error(403, self._meth +
" not supported on this resource")
def authorize(self):
if self._auth[0] == 'header':
return self.authorize_via_headertoken()
else:
return self.authorize_via_queryparam()
def authorize_via_queryparam(self):
params = parse_qs(self._env.get('QUERY_STRING', ''))
auths = params.get('auth',[])
if self._auth[1]:
# match the last value provided
return len(auths) > 0 and self._auth[1] == auths[-1]
if len(auths) > 0:
log.warning("Authorization key provided, but none has been configured")
return len(auths) == 0
def authorize_via_headertoken(self):
authhdr = self._env.get('HTTP_AUTHORIZATION', "")
log.debug("Request HTTP_AUTHORIZATION: %s", authhdr)
parts = authhdr.split()
if self._auth[1]:
return len(parts) > 1 and parts[0] == "Bearer" and \
self._auth[1] == parts[1]
if authhdr:
log.warning("Authorization key provided, but none has been configured")
return authhdr == ""
def send_unauthorized(self):
self.set_response(401, "Not authorized")
if self._auth[0] == 'header':
self.add_header('WWW-Authenticate', 'Bearer')
self.end_headers()
return []
def do_GET(self, path):
path = path.strip('/')
if not path:
try:
out = json.dumps(list(self._loaders.keys())) + '\n'
out = out.encode()
except Exception as ex:
log.exception("Internal error: "+str(ex))
return self.send_error(500, "Internal error")
self.set_response(200, "Supported Record Types")
self.add_header('Content-Type', 'application/json')
self.add_header('Content-Length', str(len(out)))
self.end_headers()
return [out]
elif path in self._loaders:
self.set_response(200, "Service is ready")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return [b"Service ready\n"]
else:
return self.send_error(404, "resource does not exist")
def do_POST(self, path):
path = path.strip('/')
steps = path.split('/')
if len(steps) == 0:
return self.send_error(405, "POST not supported on this resource")
elif len(steps) == 1:
if steps[0] == 'nerdm':
return self.ingest_nerdm_record()
else:
return self.send_error(403, "new records are not allowed for " +
"submission to this resource")
else:
return self.send_error(404, "resource does not exist")
def nerdm_archive_cache(self, rec):
"""
cache a NERDm record into a local disk archive. The cache is for
records that have been accepted but not ingested.
"""
try:
arkid = re.sub(r'/.*$', '', re.sub(r'ark:/\d+/', '', rec['@id']))
ver = rec.get('version', '1.0.0').replace('.', '_')
recid = "%s-v%s" % (os.path.basename(arkid), ver)
outfile = os.path.join(self._archdir, '_cache', recid+".json")
with open(outfile, 'w') as fd:
json.dump(rec, fd, indent=2)
return recid
except KeyError as ex:
# this shouldn't happen if the record was already validated
raise RecordIngestError("submitted record is missing the @id "+
"property")
except ValueError as ex:
# this shouldn't happen if the record was already validated
raise RecordIngestError("submitted record is apparently invalid; "+
"unable to submit")
except OSError as ex:
raise RuntimeError("Failed to cache record ({0}): {1}"
.format(arkid, str(ex)))
def nerdm_archive_commit(self, recid):
"""
commit a previously cached record to the local disk archive. This
method is called after the record has been successfully ingested to
the RMM's database.
"""
outfile = os.path.join(self._archdir, '_cache', recid+".json")
if not os.path.exists(outfile):
raise RuntimeError("record to commit ({0}) not found in cache: {1}"
.format(recid, outfile))
try:
os.rename(outfile,
os.path.join(self._archdir, os.path.basename(outfile)))
except OSError as ex:
raise RuntimeError("Failed to archvie record ({0}): {1}"
.format(recid, str(ex)))
def ingest_nerdm_record(self):
"""
Accept a NERDm record for ingest into the RMM
"""
loader = self._loaders['nerdm']
try:
clen = int(self._env['CONTENT_LENGTH'])
except KeyError as ex:
log.exception("Content-Length not provided for input record")
return self.send_error(411, "Content-Length is required")
except ValueError as ex:
log.exception("Failed to parse input JSON record: "+str(e))
return self.send_error(400, "Content-Length is not an integer")
try:
bodyin = self._env['wsgi.input']
doc = bodyin.read(clen)
rec = json.loads(doc)
except Exception as ex:
log.exception("Failed to parse input JSON record: "+str(ex))
log.warning("Input document starts...\n{0}...\n...{1} ({2}/{3} chars)"
.format(doc[:75], doc[-20:], len(doc), clen))
return self.send_error(400,
"Failed to load input record (bad format?): "+
str(ex))
try:
recid = self.nerdm_archive_cache(rec)
res = loader.load(rec, validate=True)
if res.failure_count > 0:
res = res.failures()[0]
logmsg = "Failed to load record with "+str(res.key)
for e in res.errs:
logmsg += "\n "+str(e)
log.error(logmsg)
self.set_response(400, "Input record is not valid")
self.add_header('Content-Type', 'application/json')
self.end_headers()
out = json.dumps([str(e) for e in res.errs]) + '\n'
return [ out.encode() ]
except RecordIngestError as ex:
log.exception("Failed to load posted record: "+str(ex))
self.set_response(400, "Input record is not valid (missing @id)")
self.add_header('Content-Type', 'application/json')
self.end_headers()
out = json.dumps([ "Record is missing @id property" ]) + '\n'
return [ out.encode() ]
except Exception as ex:
log.exception("Loading error: "+str(ex))
return self.send_error(500, "Load failure due to internal error")
try:
self.nerdm_archive_commit(recid)
except Exception as ex:
log.exception("Commit error: "+str(ex))
if self._postexec:
# run post-commit script
try:
self.nerdm_post_commit(recid)
except Exception as ex:
log.exception("Post-commit error: "+str(ex))
log.info("Accepted record %s with @id=%s",
rec.get('ediid','?'), rec.get('@id','?'))
self.set_response(200, "Record accepted")
self.end_headers()
return []
def nerdm_post_commit(self, recid):
"""
run an external executable for further processing after the record is commited to
the database (e.g. update an external index)
"""
cmd = _mkpostcomm(self._postexec, recid)
try:
log.debug("Executing post-commit script:\n %s", " ".join(cmd))
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
(out, err) = p.communicate()
if p.returncode != 0:
log.error("Error occurred while running post-commit script:\n"+(err or out))
except OSError as ex:
log.error("Failed to execute post-commit script:\n %s\n%s", " ".join(cmd), str(ex))
except Exception as ex:
log.error("Unexpected failure executing post-commit script:\n %s\n%s", " ".join(cmd), str(ex))
class StaticFile(object):
ACCEPT_GZIP_RE = re.compile(r'\bgzip\b')
BLOCK_SIZE = 16 * 4096
# All mimetypes starting 'text/' take a charset parameter, plus the
# additions in this set
MIMETYPES_WITH_CHARSET = {'application/javascript', 'application/xml'}
CHARSET = 'utf-8'
# Ten years is what nginx sets a max age if you use 'expires max;'
# so we'll follow its lead
FOREVER = 10*365*24*60*60
GZIP_SUFFIX = '.gz'
def __init__(self, path, is_immutable, guess_type=mimetypes.guess_type, **config):
self.path = path
stat = os.stat(path)
self.mtime_tuple = gmtime(stat.st_mtime)
mimetype, encoding = guess_type(path)
mimetype = mimetype or 'application/octet-stream'
charset = self.get_charset(mimetype)
params = {'charset': charset} if charset else {}
self.headers = Headers([
('Last-Modified', formatdate(stat.st_mtime, usegmt=True)),
('Content-Length', str(stat.st_size)),
])
self.headers.add_header('Content-Type', str(mimetype), **params)
if encoding:
self.headers['Content-Encoding'] = encoding
max_age = self.FOREVER if is_immutable else config['max_age']
if max_age is not None:
self.headers['Cache-Control'] = 'public, max-age=%s' % max_age
if config['allow_all_origins']:
self.headers['Access-Control-Allow-Origin'] = '*'
gzip_path = path + self.GZIP_SUFFIX
if os.path.isfile(gzip_path):
self.gzip_path = gzip_path
self.headers['Vary'] = 'Accept-Encoding'
# Copy the headers and add the appropriate encoding and length
self.gzip_headers = Headers(self.headers.items())
self.gzip_headers['Content-Encoding'] = 'gzip'
self.gzip_headers['Content-Length'] = str(os.stat(gzip_path).st_size)
else:
self.gzip_path = self.gzip_headers = None
def get_charset(self, mimetype):
if mimetype.startswith('text/') or mimetype in self.MIMETYPES_WITH_CHARSET:
return self.CHARSET
def serve(self, environ, start_response):
method = environ['REQUEST_METHOD']
if method != 'GET' and method != 'HEAD':
start_response('405 Method Not Allowed', [('Allow', 'GET, HEAD')])
return []
if self.file_not_modified(environ):
start_response('304 Not Modified', [])
return []
path, headers = self.get_path_and_headers(environ)
start_response('200 OK', headers.items())
if method == 'HEAD':
return []
file_wrapper = environ.get('wsgi.file_wrapper', self.yield_file)
fileobj = open(path, 'rb')
return file_wrapper(fileobj)
def file_not_modified(self, environ):
try:
last_requested = environ['HTTP_IF_MODIFIED_SINCE']
except KeyError:
return False
# Exact match, no need to parse
if last_requested == self.headers['Last-Modified']:
return True
return parsedate(last_requested) >= self.mtime_tuple
def get_path_and_headers(self, environ):
if self.gzip_path:
if self.ACCEPT_GZIP_RE.search(environ.get('HTTP_ACCEPT_ENCODING', '')):
return self.gzip_path, self.gzip_headers
return self.path, self.headers
def yield_file(self, fileobj):
# Only used as a fallback in case environ doesn't supply a
# wsgi.file_wrapper
try:
while True:
block = fileobj.read(self.BLOCK_SIZE)
if block:
yield block
else:
break
finally:
fileobj.close()
class Handler(object):
badidre = re.compile(r"[<>\s]")
def __init__(self, service, siptype, wsgienv, start_resp, auth=None):
self._svc = service
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
self._auth = auth
def send_error(self, code, message):
stat = "{0} {1}".format(str(code), message)
self._start(stat, [], sys.exc_info())
def add_header(self, name, value):
# Caution: HTTP does not support Unicode characters (see
# https://www.python.org/dev/peps/pep-0333/#unicode-issues);
# thus, this will raise a UnicodeEncodeError if the input strings
# include Unicode (char code > 255).
e = "ISO-8859-1"
self._hdr.add_header(name.encode(e), value.encode(e))
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
stat = "{0} {1}".format(str(self._code), self._msg)
self._start(stat, self._hdr.items())
def handle(self):
meth_handler = 'do_' + self._meth
path = self._env.get('PATH_INFO', '/').strip('/')
if not self.authorize():
return self.send_unauthorized()
if hasattr(self, meth_handler):
out = getattr(self, meth_handler)(path)
if isinstance(out, list) and len(out) > 0:
out.append('\n')
return out
else:
return self.send_error(
403, self._meth + " not supported on this resource")
def authorize(self):
if self._auth[0] == 'header':
return self.authorize_via_headertoken()
else:
return self.authorize_via_queryparam()
def authorize_via_queryparam(self):
params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
auths = params.get('auth', [])
if self._auth[1]:
# match the last value provided
return len(auths) > 0 and self._auth[1] == auths[-1]
if len(auths) > 0:
log.warn(
"Authorization key provided, but none has been configured")
return len(auths) == 0
def authorize_via_headertoken(self):
authhdr = self._env.get('HTTP_AUTHORIZATION', "")
parts = authhdr.split()
if self._auth[1]:
return len(parts) > 1 and parts[0] == "Bearer" and \
self._auth[1] == parts[1]
if authhdr:
log.warn(
"Authorization key provided, but none has been configured")
return authhdr == ""
def send_unauthorized(self):
self.set_response(401, "Not authorized")
if self._auth[0] == 'header':
self.add_header('WWW-Authenticate', 'Bearer')
self.end_headers()
return []
def do_GET(self, path):
# return the status on request or a list of previous requests
steps = path.split('/')
if steps[0] == '':
try:
out = json.dumps(['midas'])
except Exception, ex:
log.exception("Internal error: " + str(ex))
self.send_error(500, "Internal error")
return ["[]"]
self.set_response(200, "Supported SIP Types")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return [out]
elif steps[0] == 'midas':
if len(steps) > 2:
path = '/'.join(steps[1:])
self.send_error(400, "Unsupported SIP identifier: " + path)
return []
elif len(steps) > 1:
if steps[1].startswith("_") or steps[1].startswith(".") or \
self.badidre.search(steps[1]):
self.send_error(400, "Unsupported SIP identifier: " + path)
return []
return self.request_status(steps[1])
else:
return self.requests()
class BaseResponse:
"""Base class for Response."""
default_status = 200
default_content_type = 'text/plain;'
def __init__(self, body=None, status=None, headers=None):
self._body = body if body else [b'']
self._status_code = status or self.default_status
self.headers = Headers()
self._cookies = SimpleCookie()
if headers:
for name, value in headers.items():
self.headers.add_header(name, value)
@property
def body(self):
return self._body
@property
def status_code(self):
""" The HTTP status code as an integer (e.g. 404)."""
return self._status_code
@property
def status(self):
""" The HTTP status line as a string (e.g. ``404 Not Found``)."""
status = _HTTP_STATUS_LINES.get(self._status_code)
return str(status or ('{} Unknown'.format(self._status_code)))
@status.setter
def status(self, status_code):
if not 100 <= status_code <= 999:
raise ValueError('Status code out of range.')
self._status_code = status_code
@property
def headerlist(self):
""" WSGI conform list of (header, value) tuples. """
if 'Content-Type' not in self.headers:
self.headers.add_header('Content-Type', self.default_content_type)
if self._cookies:
for c in self._cookies.values():
self.headers.add_header('Set-Cookie', c.OutputString())
return self.headers.items()
def set_cookie(self,
key,
value,
expires=None,
max_age=None,
path='/',
secret=None,
digestmod=hashlib.sha256):
from kobin.app import current_config
if secret is None:
secret = current_config('SECRET_KEY')
if secret:
if isinstance(secret, str):
secret = secret.encode('utf-8')
encoded = base64.b64encode(
pickle.dumps((key, value), pickle.HIGHEST_PROTOCOL))
sig = base64.b64encode(
hmac.new(secret, encoded, digestmod=digestmod).digest())
value_bytes = b'!' + sig + b'?' + encoded
value = value_bytes.decode('utf-8')
self._cookies[key] = value
if len(key) + len(value) > 3800:
raise ValueError('Content does not fit into a cookie.')
if max_age is not None:
if isinstance(max_age, int):
max_age_value = max_age
else:
max_age_value = max_age.seconds + max_age.days * 24 * 3600
self._cookies[key]['max-age'] = max_age_value
if expires is not None:
if isinstance(expires, int):
expires_value = expires
else:
expires_value = time.strftime("%a, %d %b %Y %H:%M:%S GMT",
expires.timetuple())
self._cookies[key]['expires'] = expires_value
if path:
self._cookies[key]['path'] = path
def delete_cookie(self, key, **kwargs):
kwargs['max_age'] = -1
kwargs['expires'] = 0
self.set_cookie(key, '', **kwargs)
class application(object):
# don't serve static by default
static_serve = False
static_alias = { '' : 'ketcher.html' }
static_root = None
indigo = None
indigo_inchi = None
def __init__(self, environ, start_response):
self.path = environ['PATH_INFO'].strip('/')
self.method = environ['REQUEST_METHOD']
self.content_type = environ.get('CONTENT_TYPE', '')
self.fields = FieldStorage(fp=environ['wsgi.input'],
environ=environ, keep_blank_values=True)
self.FileWrapper = environ.get('wsgi.file_wrapper', FileWrapper)
self.headers = Headers([])
route = getattr(self, 'on_' + self.path, None)
if route is None:
route = self.serve_static if self.method == 'GET' and \
self.static_serve else self.notsupported
status = "200 OK"
try:
self.response = route()
except self.HttpException as e:
status = e.args[0]
self.response = [e.args[1]]
self.headers.setdefault('Content-Type', 'text/plain')
start_response(status, self.headers.items())
def __iter__(self):
for chunk in self.response:
yield chunk if sys.version_info[0] < 3 or \
not hasattr(chunk, 'encode') else chunk.encode()
def notsupported(self):
raise self.HttpException("405 Method Not Allowed",
"Request not supported")
def indigo_required(method):
def wrapper(self, **args):
if not self.indigo:
raise self.HttpException("501 Not Implemented",
"Indigo libraries are not found")
try:
return method(self, **args)
except indigo.IndigoException as e:
message = str(sys.exc_info()[1])
if 'indigoLoad' in message: # error on load
message = "Cannot load the specified " + \
"structure: %s " % str(e)
raise self.HttpException("400 Bad Request",
message)
return wrapper
@indigo_required
def on_knocknock(self):
return ["You are welcome!"]
@indigo_required
def on_layout(self):
moldata = None
if self.method == 'GET' and 'smiles' in self.fields:
moldata = self.fields.getfirst('smiles')
elif self.is_form_request() and 'moldata' in self.fields:
moldata = self.fields.getfirst('moldata')
selective = 'selective' in self.fields
if moldata:
if '>>' in moldata or moldata.startswith('$RXN'):
rxn = self.indigo.loadQueryReaction(moldata)
if selective:
for mol in rxn.iterateMolecules():
self.selective_layout(mol)
else:
rxn.layout()
return ["Ok.\n",
rxn.rxnfile()]
elif moldata.startswith('InChI'):
mol = self.indigo_inchi.loadMolecule(moldata)
mol.layout()
return ["Ok.\n",
mol.molfile()]
else:
mol = self.indigo.loadQueryMolecule(moldata)
if selective:
for rg in mol.iterateRGroups():
for frag in rg.iterateRGroupFragments():
self.selective_layout(frag)
self.selective_layout(mol)
else:
mol.layout()
return ["Ok.\n",
mol.molfile()]
self.notsupported()
@indigo_required
def on_automap(self):
moldata = None
if self.method == 'GET' and 'smiles' in self.fields:
moldata = self.fields.getfirst('smiles')
elif self.is_form_request() and 'moldata' in self.fields:
moldata = self.fields.getfirst('moldata')
if moldata:
mode = self.fields.getfirst('mode', 'discard')
rxn = self.indigo.loadQueryReaction(moldata)
if not moldata.startswith('$RXN'):
rxn.layout()
rxn.automap(mode)
return ["Ok.\n",
rxn.rxnfile()]
self.notsupported()
@indigo_required
def on_aromatize(self):
try:
md, is_rxn = self.load_moldata()
except:
message = str(sys.exc_info()[1])
if message.startswith("\"molfile loader:") and \
message.endswith("queries\""): # hack to avoid user confusion
md, is_rxn = self.load_moldata(True)
else:
raise
md.aromatize()
return ["Ok.\n",
md.rxnfile() if is_rxn else md.molfile()]
@indigo_required
def on_getinchi(self):
md, is_rxn = self.load_moldata()
inchi = self.indigo_inchi.getInchi(md)
return ["Ok.\n", inchi]
@indigo_required
def on_dearomatize(self):
try:
md, is_rxn = self.load_moldata()
except: # TODO: test for query features presence
raise self.HttpException("400 Bad Request",
"Molecules and reactions " + \
"containing query features " + \
"cannot be dearomatized yet.")
md.dearomatize()
return ["Ok.\n",
md.rxnfile() if is_rxn else md.molfile()]
def on_open(self):
if self.is_form_request():
self.headers.add_header('Content-Type', 'text/html')
return ['<html><body onload="parent.ui.loadMoleculeFromFile()" title="',
b64encode("Ok.\n"),
b64encode(self.fields.getfirst('filedata')),
'"></body></html>']
self.notsupported()
def on_save(self):
if self.is_form_request():
type, data = self.fields.getfirst('filedata').split('\n', 1)
type = type.strip()
if type == 'smi':
self.headers.add_header('Content-Type',
'chemical/x-daylight-smiles')
elif type == 'mol':
if data.startswith('$RXN'):
type = 'rxn'
self.headers.add_header('Content-Type',
'chemical/x-mdl-%sfile' % type)
self.headers.add_header('Content-Length', str(len(data)))
self.headers.add_header('Content-Disposition', 'attachment',
filename='ketcher.%s' % type)
return [data]
self.notsupported()
class HttpException(Exception): pass
def load_moldata(self, is_query=False):
moldata = self.fields.getfirst('moldata')
if moldata.startswith('$RXN'):
if is_query:
md = self.indigo.loadQueryReaction(moldata)
else:
md = self.indigo.loadReaction(moldata)
is_rxn = True
else:
if is_query:
md = self.indigo.loadQueryMolecule(moldata)
else:
md = self.indigo.loadMolecule(moldata)
is_rxn = False
return md, is_rxn
def selective_layout(self, mol):
dsgs = [dsg for dsg in mol.iterateDataSGroups() \
if dsg.description() == '_ketcher_selective_layout' and \
dsg.data() == '1']
atoms = sorted([atom.index() for dsg in dsgs \
for atom in dsg.iterateAtoms()])
for dsg in dsgs:
dsg.remove()
mol.getSubmolecule(atoms).layout()
return mol
def serve_static(self):
root = self.static_root or getcwd()
fpath = self.static_alias.get(self.path, self.path)
fpath = path.abspath(path.join(root, fpath))
if not fpath.startswith(root + path.sep) or not path.isfile(fpath) \
or fpath == path.abspath(__file__):
raise self.HttpException("404 Not Found",
"Requested file isn't accessible")
self.headers['Content-Type'] = guess_type(fpath)[0] or 'text/plain'
try:
fd = open(fpath, 'rb')
return self.FileWrapper(fd) if self.method == 'GET' else ['']
except (IOError, OSError):
raise self.HttpException("402 Payment Required", # or 403, hmm..
"Must get more money for overtime")
def is_form_request(self):
return self.method == 'POST' and \
(self.content_type.startswith('application/x-www-form-urlencoded')
or self.content_type.startswith('multipart/form-data'))
class SimRMMHandler(object):
def __init__(self, archive, wsgienv, start_resp):
self.arch = archive
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
def send_error(self, code, message):
status = "{0} {1}".format(str(code), message)
self._start(status, [], sys.exc_info())
return []
def add_header(self, name, value):
self._hdr.add_header(name, value)
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
status = "{0} {1}".format(str(self._code), self._msg)
self._start(status, self._hdr.items())
def handle(self, env, start_resp):
meth_handler = 'do_'+self._meth
path = self._env.get('PATH_INFO', '/')[1:]
params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
if hasattr(self, meth_handler):
return getattr(self, meth_handler)(path, params)
else:
return self.send_error(403, self._meth +
" not supported on this resource")
def do_GET(self, path, params=None):
if path:
path = path.rstrip('/')
if path.startswith("records"):
path = path[len("records"):].lstrip('/')
id = None
print("path="+str(path)+"; params="+str(params))
if not path and params and "@id" in params:
path = params["@id"]
path = (len(path) > 0 and path[0]) or ''
if path:
if path.startswith("ark:/88434/"):
id = path[len("ark:/88434/"):]
else:
self.arch.loadlu()
id = self.arch.ediid_to_id(path)
if id:
mdfile = os.path.join(self.arch.dir, id+".json")
if not id or not os.path.exists(mdfile):
if not id:
id = "resource"
return self.send_error(404, id + " does not exist")
try:
with open(mdfile) as fd:
data = json.load(fd, object_pairs_hook=OrderedDict)
data["_id"] ={"timestamp":1521220572,"machineIdentifier":3325465}
if params and "@id" in params:
data = { "ResultCount": 1, "PageSize": 0,
"ResultData": [ data ] }
except Exception as ex:
print(str(ex))
return self.send_error(500, "Internal error")
self.set_response(200, "Identifier exists")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return [ json.dumps(data, indent=2) + "\n" ]
class Handler(object):
badidre = re.compile(r"[<>\s]")
def __init__(self, service, filemap, wsgienv, start_resp):
self._svc = service
self._fmap = filemap
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
def send_error(self, code, message):
status = "{0} {1}".format(str(code), message)
self._start(status, [], sys.exc_info())
def add_header(self, name, value):
# Caution: HTTP does not support Unicode characters (see
# https://www.python.org/dev/peps/pep-0333/#unicode-issues);
# thus, this will raise a UnicodeEncodeError if the input strings
# include Unicode (char code > 255).
e = "ISO-8859-1"
self._hdr.add_header(name.encode(e), value.encode(e))
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
status = "{0} {1}".format(str(self._code), self._msg)
###DEBUG:
log.debug("sending header: %s", str(self._hdr.items()))
###DEBUG:
self._start(status, self._hdr.items())
def handle(self):
meth_handler = 'do_' + self._meth
path = self._env.get('PATH_INFO', '/')[1:]
if hasattr(self, meth_handler):
return getattr(self, meth_handler)(path)
else:
return self.send_error(
403, self._meth + " not supported on this resource")
def do_GET(self, path):
if not path:
self.code = 403
self.send_error(self.code, "No identifier given")
return ["Server ready\n"]
if path.startswith('/'):
path = path[1:]
parts = path.split('/')
if parts[0] == "ark:":
# support full ark identifiers
if len(parts) > 2 and parts[1] == NIST_ARK_NAAN:
dsid = parts[2]
else:
dsid = '/'.join(parts[:3])
filepath = "/".join(parts[3:])
else:
dsid = parts[0]
filepath = "/".join(parts[1:])
if self.badidre.search(dsid):
self.send_error(400, "Unsupported SIP identifier: " + dsid)
return []
if filepath:
return self.get_datafile(dsid, filepath)
return self.get_metadata(dsid)
def get_metadata(self, dsid):
try:
mdata = self._svc.resolve_id(dsid)
except IDNotFound as ex:
self.send_error(404,
"Dataset with ID={0} not available".format(dsid))
return []
except SIPDirectoryNotFound as ex:
# shouldn't happen
self.send_error(404,
"Dataset with ID={0} not available".format(dsid))
return []
except Exception as ex:
log.exception("Internal error: " + str(ex))
self.send_error(500, "Internal error")
return []
self.set_response(200, "Identifier found")
self.add_header('Content-Type', 'application/json')
self.end_headers()
return [json.dumps(mdata, indent=4, separators=(',', ': '))]
def get_datafile(self, id, filepath):
try:
loc, mtype = self._svc.locate_data_file(id, filepath)
except IDNotFound as ex:
self.send_error(404,
"Dataset with ID={0} not available".format(id))
return []
except SIPDirectoryNotFound as ex:
# shouldn't happen
self.send_error(404,
"Dataset with ID={0} not available".format(id))
return []
except Exception as ex:
log.exception("Internal error: " + str(ex))
self.send_error(500, "Internal error")
return []
if not loc:
self.send_error(
404, "Dataset (ID={0}) does not contain file={1}".format(
id, filepath))
xsend = None
prfx = [p for p in self._fmap.keys() if loc.startswith(p + '/')]
if len(prfx) > 0:
xsend = self._fmap[prfx[0]] + loc[len(prfx[0]):]
log.debug("Sending file via X-Accel-Redirect: %s", xsend)
self.set_response(200, "Data file found")
self.add_header('Content-Type', mtype)
if xsend:
self.add_header('X-Accel-Redirect', xsend)
self.end_headers()
if xsend:
return []
return self.iter_file(loc)
def iter_file(self, loc):
# this is the backup, inefficient way to send a file
with open(loc, 'rb') as fd:
buf = fd.read(5000000)
yield buf
def do_HEAD(self, path):
self.do_GET(path)
return []
def make_headers(self, headers):
h = Headers([("Allow", "GET, HEAD")])
for item in headers:
h.add_header(item[0], item[1])
return h.items()
class SimDistribHandler(object):
def __init__(self, archive, wsgienv, start_resp):
self.arch = archive
self._env = wsgienv
self._start = start_resp
self._meth = wsgienv.get('REQUEST_METHOD', 'GET')
self._hdr = Headers([])
self._code = 0
self._msg = "unknown status"
def send_error(self, code, message):
status = "{0} {1}".format(str(code), message)
self._start(status, [], sys.exc_info())
return []
def add_header(self, name, value):
self._hdr.add_header(name, value)
def set_response(self, code, message):
self._code = code
self._msg = message
def end_headers(self):
status = "{0} {1}".format(str(self._code), self._msg)
self._start(status, self._hdr.items())
def handle(self, env, start_resp):
meth_handler = 'do_' + self._meth
path = self._env.get('PATH_INFO', '/')[1:]
params = cgi.parse_qs(self._env.get('QUERY_STRING', ''))
if hasattr(self, meth_handler):
return getattr(self, meth_handler)(path, params)
else:
return self.send_error(
403, self._meth + " not supported on this resource")
def do_HEAD(self, path, params=None, forhead=False):
return self.do_GET(path, params, True)
def do_GET(self, path, params=None, forhead=False):
aid = None
vers = None
path = path.strip('/')
if path.startswith("od/ds/"):
path = path[len("od/ds/"):]
print("processing " + path)
# refresh the archive
self.arch.loadinfo()
if not path:
try:
out = json.dumps(self.arch.aipids) + '\n'
except Exception as ex:
return self.send_error(500, "Internal error")
self.set_response(200, "AIP Identifiers")
self.add_header('Content-Type', 'application/json')
self.add_header('Content-Length', str(len(out)))
self.end_headers()
if forhead:
return []
return [out]
elif path.startswith("_aip/"):
# requesting a bag file
path = path[len("_aip/"):].strip('/')
filepath = os.path.join(self.arch.dir, path)
if os.path.isfile(filepath):
self.set_response(200, "Bag file found")
self.add_header('Content-Type', "application/zip")
self.end_headers()
if forhead:
return []
return self.iter_file(filepath)
else:
return self.send_error(404, "bag file does not exist")
elif '/' in path:
parts = path.split('/', 1)
aid = parts[0]
path = (len(parts) > 1 and parts[1]) or ''
print("accessing " + aid)
elif path:
aid = path
path = ''
else:
return self.send_error(404, "resource does not exist")
# path-info is now captured as aid and path
if aid not in self.arch._aips:
return self.send_error(404, "resource does not exist")
if not path:
self.set_response(200, "AIP Identifier exists")
self.add_header('Content-Type', 'application/json')
self.add_header('Content-Length', str(len(aid) + 4))
self.end_headers()
if forhead:
return []
return ['["' + aid + '"]']
elif path == "_aip":
try:
out = json.dumps(self.arch.list_bags(aid)) + '\n'
except Exception, ex:
return self.send_error(500, "Internal error")
self.set_response(200, "All bags for ID")
self.add_header('Content-Type', 'application/json')
self.add_header('Content-Length', str(len(out)))
self.end_headers()
if forhead:
return []
return [out]
elif path == "_aip/_head":
try:
out = self.arch.head_for(aid)
if out:
out = json.dumps(out) + '\n'
except Exception, ex:
print(
"Failed to create JSON output for head bag, aid={0}: {2}".
format(aid, vers, str(ex)))
return self.send_error(500, "Internal error")