async def forward(request, url):
#print('>', url)
headers = {'Accept': 'application/vnd.phereo.v3+json'}
for k in ('Cache-Control', 'If-Modified-Since', 'If-None-Match',
'User-Agent'):
if k in request.headers:
headers[k] = request.headers[k]
async with ClientSession() as client:
async with client.request('GET',
url,
headers=headers,
allow_redirects=False,
ssl=False,
data=await request.read()) as res:
if res.status == 404:
raise web.HTTPNotFound()
elif res.status == 302:
raise web.HTTPFound(location=res.headers.get('Location'))
elif res.status == 304:
raise web.HTTPNotModified()
elif res.status != 200:
raise web.HTTPInternalServerError() # Not expected
headers = {'Access-Control-Allow-Origin': '*'}
for k in ('Content-Type', 'Expires', 'Cache-Control', 'Pragma',
'ETag', 'Last-Modified'):
if k in res.headers:
headers[k] = res.headers[k]
return web.Response(status=200,
headers=headers,
body=await res.read())
async def link_redirect(request: web.Request):
dataset = request.match_info['dataset']
distribution = request.match_info['distribution']
etag_if_none_match = conditional.parse_if_header(
request, conditional.HEADER_IF_NONE_MATCH)
if etag_if_none_match == '*':
raise web.HTTPBadRequest(
body='Endpoint does not support * in the If-None-Match header.')
try:
doc, etag = await request.app.hooks.storage_retrieve(
app=request.app, docid=dataset, etags=etag_if_none_match)
except KeyError:
raise web.HTTPNotFound()
if doc is None:
raise web.HTTPNotModified(headers={'ETag': etag})
headers = {'ETag': etag}
resource_url = None
for dist in doc.get('dcat:distribution', []):
if dist.get('dc:identifier', None) == distribution:
resource_url = dist.get('dcat:accessURL', None)
break
if resource_url is None:
raise web.HTTPNotFound(headers=headers)
raise web.HTTPTemporaryRedirect(location=resource_url, headers=headers)
async def patch_message(req: web.Request) -> web.Response:
message_id = req["match_info"]["message_id"]
params = {k: v for k, v in req["body"].items() if v is not None}
if not params:
raise web.HTTPNotModified()
record = await req.config_dict["pg_conn"].fetchval(
MESSAGE.update_query_for(
"messages", message_id, params.keys(), returning=False
),
*params.values(),
req.config_dict["sf_gen"].gen_id(),
)
if record is None:
raise web.HTTPNotModified
# FIXME: reduce number of queries!!! (3 here)
new_row = await req.config_dict["pg_conn"].fetchrow(
f"SELECT {MESSAGE} FROM messages_with_author WHERE id = $1", message_id
)
diff = MESSAGE.diff_to_json(req["message"], new_row)
req.config_dict["emitter"].emit(events.MESSAGE_UPDATE(payload=diff))
return web.json_response(diff)
async def add_channel_recipient(req: web.Request) -> web.Response:
await helpers.ensure_permissions(Permissions.INVITE_MEMBERS, request=req)
channel_id = req["match_info"]["channel_id"]
user_id = req["match_info"]["user_id"]
await ensure_existance(req, "users", user_id, "User")
async with req.config_dict["pg_conn"].acquire() as conn:
async with conn.transaction():
success = await conn.fetchval(
"SELECT * FROM add_channel_user($1, $2)", channel_id, user_id
)
if not success:
raise web.HTTPNotModified(reason="Is user already in channel?")
message = await conn.fetchrow(
f"SELECT {MESSAGE} FROM create_message($1, $2, $3, $4, type:=$5)",
req.config_dict["sf_gen"].gen_id(),
channel_id,
user_id,
"",
MessageTypes.RECIPIENT_ADD.value,
)
req.config_dict["emitter"].emit(
events.MESSAGE_CREATE(payload=MESSAGE.to_json(message))
)
raise web.HTTPNoContent()
def _assert_if_none_match(request: web.Request, etag: T.Union[None, bool, str],
require: bool):
# The If-None-Match: header is used in two scenarios:
# 1. GET requests by a caching client. In this case, the client will
# normally provide a list of (cached) ETags.
# 2. PUT requests, where the client intends to create a new resource and
# wants to avoid overwriting an existing resource. In this case, the
# client will normally provide only the asterisk "*" character.
etags = _parse_if_header(request, 'If-None-Match')
if require and etags is None:
raise web.HTTPPreconditionRequired(text='If-None-Match')
if etags is None or etag is False or etag is None:
return
if etags is _STAR:
raise web.HTTPPreconditionFailed()
# From here on, we know that etags is a set of strings.
if etag is True:
raise web.HTTPPreconditionFailed(
text="Resource doesn't support ETags.")
# From here on, we know that etag is a string:
if etag in etags:
if request.method in {'GET', 'HEAD'}:
raise web.HTTPNotModified()
else:
raise web.HTTPPreconditionFailed()
def img(request):
""" return file and information about it, for static """
proc_id = request.match_info.get('proc_id', "des:obj")
doc_id = request.match_info.get('doc_id', "")
img = request.match_info.get('img', "")
action = request.match_info.get('action', "img")
att = None
headers = {}
try:
fn, att, prefix = img_m(request, proc_id, doc_id, img, action)
if not fn: return web.HTTPNotFound()
headers['Content-Length'] = fn['length']
lm = locale_date("%a, %d %b %Y %H:%M:%S GMT", fn['uploadDate'].timetuple(), 'en_US.UTF-8')
headers['Last-Modified'] = lm
ims = request.if_modified_since
if ims and ims.strftime('%Y-%m-%d %H:%M:%S') >= fn['uploadDate'].strftime('%Y-%m-%d %H:%M:%S'):
headers['Date'] = locale_date("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(), 'en_US.UTF-8')
return web.HTTPNotModified( headers=MultiDict( headers ) )
headers['Content-Type'] = fn['mime']
headers['Cache-Control'] = 'max-age=604800'
content = att.read()
finally:
if att: att.close()
return web.Response(body=content, headers=MultiDict( headers ))
async def get_user_status_list(req):
date_etag_separator = ' || '
user_id =req.match_info['user_id']
session = req.app[HTTP_SESSION]
if not user_id:
return web.HTTPBadRequest(text='Expected /user_status/list/{user_id}')
goodreads_url = 'https://www.goodreads.com/user_status/list/{}?format=rss'.format(user_id)
headers = {}
try:
logger.info('Checking cache..')
with open('./data/{}'.format(user_id), 'r') as cache:
data = cache.readline()
lastmodm, last_etag = data.strip().split(date_etag_separator)
headers['If-Modified-Since'] = lastmodm
headers['If-None-Match'] = last_etag
except FileNotFoundError:
pass
async with session.get(goodreads_url, headers=headers) as resp:
if resp.status == 304:
return web.HTTPNotModified()
last_modified = resp.headers.get('last-modified', '')
etag = resp.headers.get('etag', '')
if last_modified and etag:
with open('./data/{}'.format(user_id), 'w') as cache:
cache.write('{}{}{}'.format(last_modified, date_etag_separator, etag))
raw_feed = await resp.text()
new_feed = transform_feed(raw_feed.encode('utf-8'))
return web.Response(body=new_feed, content_type='application/rss+xml')
def _assert_if_none_match(request: web.Request, etag: T.Union[None, bool, str],
allow_weak: bool) -> bool:
# language=rst
"""
Todo:
Evert lammerts schreef: Documenteren. Ik vind _assert_if_none_match niet
makkelijk te lezen zonder docs voor de parameters. Bv: etag heeft als
type None, bool, of str. En if etag is True: PreconditionFailed. Die
semantiek is vast logisch in context maar niet op zichzelf.
The If-None-Match: header is used in two scenarios:
1. GET requests by a caching client. In this case, the client will
normally provide a list of (cached) ETags.
2. PUT requests, where the client intends to create a new resource and
wants to avoid overwriting an existing resource. In this case, the
client will normally provide only the asterisk "*" character.
Returns:
bool: indicates if an ``If-None-Match`` header was provided.
Raises:
web.HTTPBadRequest:
If the request header is syntactically incorrect.
web.HTTPPreconditionRequired:
If parameter `require` is ``True`` and no precondition was provided
by the client.
web.HTTPPreconditionFailed:
If the precondition failed and the request is *unsafe*.
web.HTTPNotModified:
If the precondition failed and the request is *safe*.
"""
etags = _parse_if_header(request, _IF_NONE_MATCH)
if etags is None:
# if require and request.method in unsafe_methods:
# raise web.HTTPPreconditionRequired(text=_IF_NONE_MATCH)
return False
if etag is False or etag is None:
return True
if etags is _STAR:
raise web.HTTPPreconditionFailed(text=_IF_NONE_MATCH)
# From here on, we know that etags is a set of strings.
if etag is True:
raise web.HTTPPreconditionFailed(text="Resource doesn't have an ETag.")
# From here on, we know that etag is a string:
if _match_etags(etag, etags, allow_weak):
if request.method in {hdrs.METH_GET, hdrs.METH_HEAD}:
raise web.HTTPNotModified()
else:
raise web.HTTPPreconditionFailed(text=_IF_NONE_MATCH)
return True
async def add_pin(req: web.Request) -> web.Response:
channel_id = req["match_info"]["channel_id"]
message_id = req["match_info"]["message_id"]
async with req.config_dict["pg_conn"].acquire() as conn:
async with conn.transaction():
pins_ids = await conn.fetchval(
"SELECT pinned_ids FROM channels WHERE id = $1", channel_id
)
if len(pins_ids) >= 50:
raise web.HTTPBadRequest(reason="Too many pins (>= 50)")
if message_id in pins_ids:
raise web.HTTPNotModified(reason="Already pinned")
pin_success = await conn.fetchval(
"SELECT * FROM add_channel_pin($1, $2)", message_id, channel_id
)
if not pin_success:
raise web.HTTPBadRequest(
reason="Failed to pin message. Does it belong to channel?"
)
message = await conn.fetchrow(
f"SELECT {MESSAGE} FROM create_message($1, $2, $3, $4, type:=$5)",
req.config_dict["sf_gen"].gen_id(),
channel_id,
req["access_token"].user_id,
"",
MessageTypes.CHANNEL_PIN_ADD.value,
)
req.config_dict["emitter"].emit(
events.MESSAGE_CREATE(payload=MESSAGE.to_json(message))
)
raise web.HTTPNoContent()
def _handler(request):
try:
res = yield from handler(request)
except web.HTTPClientError as e:
if Config.app.debug:
error = traceback.format_exc()
error_page = DEFAULT_ERROR_PRINT_PAGE % (error.replace(
'\n', '<br/> '))
logging.error(error)
else:
try:
error_template = 'errors/' + \
str(e.status_code) + '.html'
error_page = app.get('__templating__').render(
error_template)
except jinja2.exceptions.TemplateNotFound:
error_page = DEFAULT_HTTP_ERROR_PAGE % (
e.status_code, "Pyblog 1.0", e.status_code,
e.reason)
res = web.Response(status=e.status_code,
body=error_page.encode("utf-8"))
except web.HTTPServerError as e:
if Config.app.debug:
error = traceback.format_exc()
error_page = DEFAULT_ERROR_PRINT_PAGE % (error.replace(
'\n', '<br/> '))
logging.error(error)
else:
try:
error_template = 'errors/' + \
str(e.status_code) + '.html'
error_page = app.get('__templating__').render(
error_template)
except jinja2.exceptions.TemplateNotFound:
error_page = DEFAULT_HTTP_ERROR_PAGE % (
e.status_code, "Pyblog 1.0", e.status_code,
e.reason)
res = web.Response(status=e.status_code,
body=error_page.encode("utf-8"))
except web.HTTPException as e:
if int(e.status_code) == 304:
return web.HTTPNotModified()
if Config.app.debug:
error = traceback.format_exc()
error_page = DEFAULT_ERROR_PRINT_PAGE % (error.replace(
'\n', '<br/> '))
logging.error(error)
else:
error_page = DEFAULT_HTTP_ERROR_PAGE % (
e.status_code, "Pyblog 1.0", e.status_code, e.reason)
res = web.Response(status=e.status_code,
body=error_page.encode('utf-8'))
except Exception as e:
if Config.app.debug:
error = traceback.format_exc()
error_page = DEFAULT_ERROR_PRINT_PAGE % (error.replace(
'\n', '<br/> '))
logging.error(error)
else:
error_page = DEFAULT_HTTP_ERROR_PAGE % (
500, "Pyblog 1.0", 500, "Server Internal Error")
res = web.Response(status=500, body=error_page.encode("utf-8"))
else:
if app.get('status'):
res = web.Response(
status=app.get('status').get('code'),
body=app.get('status').get('message').encode('utf-8'))
return res
return res
def view(user, page: str, if_modified_since) -> web.Response:
if page.endswith("/"):
page += "Main Page"
wiki_page = WikiPage(page)
page_error = wiki_page.page_is_valid(page)
if page_error:
return error.view(user, page, page_error)
# If there is a difference in case, nicely point this out to users.
correct_page = wiki_page.page_get_correct_case(page)
if correct_page != page:
return error.view(
user,
page,
f'"{page}" does not exist; did you mean [[{correct_page}]]?',
)
status_code = 200 if wiki_page.page_exists(page) else 404
namespaced_page = page
if not namespaced_page.startswith(("Category/", "File/", "Folder/", "Template/")):
namespaced_page = f"Page/{namespaced_page}"
can_cache = status_code == 200 and not namespaced_page.startswith("Folder/")
if CACHE_PAGE_FOLDER:
cache_filename = f"{CACHE_PAGE_FOLDER}/{namespaced_page}.html"
else:
cache_filename = None
response = None
# Check as we might have this page already on cache.
if can_cache and namespaced_page in metadata.LAST_TIME_RENDERED:
if (
if_modified_since is not None
and metadata.LAST_TIME_RENDERED[namespaced_page] <= if_modified_since.timestamp()
):
# We already rendered this page before. If the browser has it in his
# cache, he can simply reuse that if we haven't rendered since.
response = web.HTTPNotModified()
elif (
not user
and cache_filename
and os.path.exists(cache_filename)
and os.path.getmtime(cache_filename) >= metadata.LAST_TIME_RENDERED[namespaced_page]
):
# We already rendered this page to disk. Serve from there.
with open(cache_filename) as fp:
body = fp.read()
response = web.Response(body=body, content_type="text/html", status=status_code)
# Cache miss; render the page.
if response is None:
body = _view(wiki_page, user, page)
# Never cache anything in the Folder/.
if can_cache:
if not user and cache_filename:
# Cache the file on disk
os.makedirs(os.path.dirname(cache_filename), exist_ok=True)
with open(cache_filename, "w") as fp:
fp.write(body)
page_time = os.path.getmtime(cache_filename)
else:
# Accuracy of time.time() is higher than getmtime(), so
# depending if we cached, use a different clock.
page_time = time.time()
# Only update the time if we don't have one yet. This makes sure
# that LAST_TIME_RENDERED has the oldest timestamp possible.
if namespaced_page not in metadata.LAST_TIME_RENDERED:
metadata.LAST_TIME_RENDERED[namespaced_page] = page_time
response = web.Response(body=body, content_type="text/html", status=status_code)
# Inform the browser under which rules it can cache this page.
if can_cache:
response.last_modified = metadata.LAST_TIME_RENDERED[namespaced_page]
response.headers["Vary"] = "Accept-Encoding, Cookie"
response.headers["Cache-Control"] = "private, must-revalidate, max-age=0"
return response