async def middleware(
request: web.Request, handler: Handler
) -> web.StreamResponse:
# Initial vars
request_method = request.method
request_path = request.rel_url.path
# Is this an OPTIONS request
is_options_request = request_method == "OPTIONS"
# Is this a preflight request
is_preflight_request = (
is_options_request
and ACCESS_CONTROL_REQUEST_METHOD in request.headers
)
# Log extra data
log_extra = {
"is_preflight_request": is_preflight_request,
"method": request_method.lower(),
"path": request_path,
}
# Check whether CORS should be enabled for given URL or not. By default
# CORS enabled for all URLs
if not match_items(check_urls, request_path):
logger.debug(
"Request should not be processed via CORS middleware",
extra=log_extra,
)
return await handler(request)
# If this is a preflight request - generate empty response
if is_preflight_request:
response = web.StreamResponse()
# Otherwise - call actual handler
else:
response = await handler(request)
# Now check origin heaer
origin = request.headers.get("Origin")
# Empty origin - do nothing
if not origin:
logger.debug(
"Request does not have Origin header. CORS headers not "
"available for given requests",
extra=log_extra,
)
return response
# Set allow credentials header if necessary
if allow_credentials:
response.headers[ACCESS_CONTROL_ALLOW_CREDENTIALS] = "true"
# Check whether current origin satisfies CORS policy
if not allow_all and not (origins and match_items(origins, origin)):
logger.debug(
"CORS headers not allowed for given Origin", extra=log_extra
)
return response
# Now start supplying CORS headers
# First one is Access-Control-Allow-Origin
if allow_all and not allow_credentials:
cors_origin = "*"
else:
cors_origin = origin
response.headers[ACCESS_CONTROL_ALLOW_ORIGIN] = cors_origin
# Then Access-Control-Expose-Headers
if expose_headers:
response.headers[ACCESS_CONTROL_EXPOSE_HEADERS] = ", ".join(
expose_headers
)
# Now, if this is an options request, respond with extra Allow headers
if is_options_request:
response.headers[ACCESS_CONTROL_ALLOW_HEADERS] = ", ".join(
allow_headers
)
response.headers[ACCESS_CONTROL_ALLOW_METHODS] = ", ".join(
allow_methods
)
if max_age is not None:
response.headers[ACCESS_CONTROL_MAX_AGE] = str(max_age)
# If this is preflight request - do not allow other middlewares to
# process this request
if is_preflight_request:
logger.debug(
"Provide CORS headers with empty response for preflight "
"request",
extra=log_extra,
)
raise web.HTTPOk(text="", headers=response.headers)
# Otherwise return normal response
logger.debug("Provide CORS headers for request", extra=log_extra)
return response
async def handler(request):
cloned = request.clone()
resp = web.StreamResponse()
await resp.prepare(cloned)
return resp
async def stream_response(request):
session = await get_session(request)
session['will_not'] = 'show up'
return web.StreamResponse()
def process(self):
session = self.session
request = self.request
meth = request.method
if request.method == hdrs.METH_GET:
try:
callback = self.callback = request.query.get('c')
except Exception:
callback = self.callback = request.GET.get('c')
if not callback:
yield from self.session._remote_closed()
return web.HTTPInternalServerError(
body=b'"callback" parameter required')
elif not self.check_callback.match(callback):
yield from self.session._remote_closed()
return web.HTTPInternalServerError(
body=b'invalid "callback" parameter')
headers = list((
(hdrs.CONTENT_TYPE, 'application/javascript; charset=UTF-8'),
(hdrs.CACHE_CONTROL, CACHE_CONTROL)) +
session_cookie(request) +
cors_headers(request.headers))
resp = self.response = web.StreamResponse(headers=headers)
yield from resp.prepare(request)
yield from self.handle_session()
return resp
elif request.method == hdrs.METH_POST:
data = yield from request.read()
ctype = request.content_type.lower()
if ctype == 'application/x-www-form-urlencoded':
if not data.startswith(b'd='):
return web.HTTPInternalServerError(
body=b'Payload expected.')
data = unquote_plus(data[2:].decode(ENCODING))
else:
data = data.decode(ENCODING)
if not data:
return web.HTTPInternalServerError(body=b'Payload expected.')
try:
messages = loads(data)
except Exception:
return web.HTTPInternalServerError(
body=b'Broken JSON encoding.')
yield from session._remote_messages(messages)
return web.Response(
body=b'ok',
headers=((hdrs.CONTENT_TYPE, 'text/plain; charset=UTF-8'),
(hdrs.CACHE_CONTROL, CACHE_CONTROL)) +
session_cookie(request))
else:
return web.HTTPBadRequest(text="No support for such method: %s" %
meth)
async def get_collection(request: web.Request) -> web.StreamResponse:
# language=rst
"""Handler for ``/datasets``"""
hooks = request.app.hooks
query = request.query
scopes = request.authz_scopes if hasattr(request, "authz_scopes") else {}
extra_read_access = 'CAT/R' in scopes
# show non-available datasets only to extra_read_access
if extra_read_access:
filters = {}
else:
filters = {
'/properties/ams:status': {
'in': ['beschikbaar', 'in_onderzoek']
}
}
# Extract facet filters:
for key in query:
if not _FACET_QUERY_KEY.fullmatch(key) or (
not extra_read_access and key == '/properties/ams:status'):
continue
if key not in filters:
filters[key] = {}
match: T.List[str] = _FACET_QUERY_VALUE.fullmatch(query[key])
if not match:
raise web.HTTPBadRequest(
text="Unknown comparator in query parameter '%s'" % key)
comparator = match[1]
value = match[2]
if comparator in filters[key]:
raise web.HTTPBadRequest(
text="Multiple facet filters for facet %s with comparator %s" %
(key, comparator))
if comparator == 'in':
value = _csv_decode_line(value)
if value is None:
raise web.HTTPBadRequest(
text=
"Value of query parameter '%s' is not a CSV encoded list of strings; see RFC4180"
% key)
filters[key][comparator] = value
full_text_query = query.get('q', '').strip()
limit = query.get('limit', None)
if limit is not None:
try:
limit = int(limit)
except ValueError:
raise web.HTTPBadRequest(text="Invalid limit value %s" % limit)
offset = query.get('offset', 0)
if offset is not None:
try:
offset = int(offset)
except ValueError:
raise web.HTTPBadRequest(text="Invalid offset value %s" % offset)
result_info = {}
facets = [
'/properties/dcat:distribution/items/properties/ams:resourceType',
'/properties/dcat:distribution/items/properties/dcat:mediaType',
'/properties/dcat:distribution/items/properties/ams:distributionType',
'/properties/dcat:distribution/items/properties/ams:serviceType',
'/properties/dcat:keyword/items', '/properties/dcat:theme/items',
'/properties/ams:owner'
]
if extra_read_access:
facets.append('/properties/ams:status')
resultiterator = await hooks.search_search(app=request.app,
q=full_text_query,
sortpath=['ams:sort_modified'],
result_info=result_info,
facets=facets,
limit=limit,
offset=offset,
filters=filters,
iso_639_1_code='nl')
ctx = await hooks.mds_context()
ctx_json = json.dumps(ctx)
response = web.StreamResponse()
response.content_type = request['best_content_type']
response.enable_compression()
first = True
await response.prepare(request)
await response.write(b'{"@context":')
await response.write(ctx_json.encode())
await response.write(b',"dcat:dataset":[')
async for docid, doc in resultiterator:
canonical_doc = await hooks.mds_canonicalize(app=request.app, data=doc)
canonical_doc = await hooks.mds_after_storage(app=request.app,
data=canonical_doc,
doc_id=docid)
keepers = {
'@id', 'dct:identifier', 'dct:title', 'dct:description',
'dcat:keyword', 'foaf:isPrimaryTopicOf', 'dcat:distribution',
'dcat:theme', 'ams:owner', 'ams:sort_modified'
}
if extra_read_access:
keepers.add('ams:status')
for key in list(canonical_doc.keys()):
if key not in keepers:
del canonical_doc[key]
keepers = {
'dcat:mediaType', 'ams:resourceType', 'ams:distributionType',
'ams:serviceType', 'dc:identifier'
}
for d in canonical_doc.get('dcat:distribution', []):
for key in list(d.keys()):
if key not in keepers:
del d[key]
if not first:
await response.write(b',')
else:
first = False
await response.write(json.dumps(canonical_doc).encode())
await response.write(b']')
await response.write(b', "void:documents": ')
await response.write(str(result_info['/']).encode())
del result_info['/']
await response.write(b', "ams:facet_info": ')
await response.write(json.dumps(result_info).encode())
await response.write(b'}')
await response.write_eof()
return response
async def binary(self, data, type='application/octet-stream'):
self.response = web.StreamResponse()
self.response.content_length = len(data)
self.response.content_type = type
await self.response.prepare(self.request)
self.response.write(data)
async def ledger_text(request):
if not TRUST_ANCHOR.ready:
return not_ready()
response = web.StreamResponse()
response.content_type = "text/plain"
response.charset = "utf-8"
await response.prepare(request)
rows = await TRUST_ANCHOR.get_txn_range(request.match_info["ledger_name"])
first = True
for seq_no, added, row in rows:
text = []
if not first:
text.append("")
first = False
row = json.loads(row)
txn = row["txn"]
data = txn["data"]
metadata = txn["metadata"]
type_name = INDY_TXN_TYPES.get(txn['type'], txn['type'])
text.append("[" + str(seq_no) + "] TYPE: " + type_name)
ident = metadata.get('from')
if ident != None:
text.append("FROM: " + ident)
if type_name == "NYM":
text.append("DEST: " + data['dest'])
role = data.get('role')
if role != None:
role_name = INDY_ROLE_TYPES.get(role, role)
text.append("ROLE: " + role_name)
verkey = data.get('verkey')
if verkey != None:
text.append("VERKEY: " + verkey)
txnTime = txn.get('txnTime')
if txnTime != None:
ftime = datetime.fromtimestamp(txnTime).strftime(
'%Y-%m-%d %H:%M:%S')
text.append("TIME: " + ftime)
reqId = metadata.get('reqId')
if reqId != None:
text.append("REQ ID: " + str(reqId))
refNo = data.get('ref')
if refNo != None:
text.append("REF: " + str(refNo))
txnId = row['txnMetadata'].get('txnId')
if txnId != None:
text.append("TXN ID: " + txnId)
if type_name == "SCHEMA" or type_name == "CLAIM_DEF" or type_name == "NODE":
data = data.get('data')
text.append("DATA:")
text.append(json.dumps(data, indent=4))
sig = data.get('signature')
if sig != None:
text.append("SIGNATURE: " + sig)
sig_type = data.get('signature_type')
if sig_type != None:
text.append("SIGNATURE TYPE: " + sig_type)
text.append("")
await response.write("\n".join(text).encode("utf-8"))
await response.write_eof()
return response
async def game_stream(request):
auth = request.headers.get("Authorization")
if auth is None:
return web.HTTPForbidden()
token = auth[auth.find("Bearer") + 7:]
if token not in BOT_TOKENS:
log.error("BOT account auth with token %s failed", token)
return web.HTTPForbidden()
user_agent = request.headers.get("User-Agent")
username = user_agent[user_agent.find("user:"******"gameId"]
users = request.app["users"]
games = request.app["games"]
game = games[gameId]
resp = web.StreamResponse()
resp.content_type = "application/json"
await resp.prepare(request)
bot_player = users[username]
log.info("+++ %s connected to %s game stream", bot_player.username, gameId)
await bot_player.game_queues[gameId].put(game.game_full)
async def pinger():
""" To help lichess-bot.py abort games showing no activity. """
while True:
if gameId in bot_player.game_queues:
await bot_player.game_queues[gameId].put("\n")
await asyncio.sleep(5)
else:
break
pinger_task = asyncio.create_task(pinger())
while True:
answer = await bot_player.game_queues[gameId].get()
try:
bot_player.game_queues[gameId].task_done()
except ValueError:
log.error(
"task_done() called more times than there were items placed in the queue in bot_api.py game_stream()"
)
try:
await resp.write(answer.encode("utf-8"))
await resp.drain()
except Exception:
log.error("Writing %s to BOT game_stream failed!", answer)
break
try:
await resp.write_eof()
except Exception:
log.error("Writing EOF to BOT game_stream failed!")
pinger_task.cancel()
return resp
async def guilds(self, request: web.Request):
token = request.query.get("token")
guild_id = request.query.get("guild_id")
if not (token and guild_id):
raise web.HTTPFound("/register")
if not guild_id.isdigit():
raise web.HTTPNotFound()
guild_id = int(guild_id)
medata = await (await self.session.get(
"https://discordapp.com/api/users/@me",
headers={
"Authorization": f"Bearer {token}",
})).json()
guilds = await (await self.session.get(
"https://discordapp.com/api/users/@me/guilds",
headers={
"Authorization": f"Bearer {token}",
})).json()
guild = self.bot.get_guild(guild_id)
if "code" in guilds:
return web.StreamResponse(reason=guilds["message"],
status=guilds["code"])
if str(guild_id) not in (g["id"] for g in guilds):
raise web.HTTPForbidden()
try:
guild_data = await self.get_serverdata(guild_id)
user_data = (await self.get_userdata(medata["id"]))[str(guild_id)]
except:
import traceback
traceback.print_exc()
raise web.HTTPBadRequest(reason="oof")
html = self.guild_html
start = "Start Money: {}".format(guild_data["start"])
stats = "Balance: {}\n<br />Level: {}\n<br />Exp: {}".format(
user_data["money"], user_data.get("level"), user_data.get("exp"))
fmap = map(lambda x: f"<li>{x[0]} x{x[1]}</li>",
sorted(user_data["items"].items()))
inventory = "\n".join(fmap)
req = f"""SELECT (UUID, info->'{guild_id}'->>'money') FROM userdata;"""
async with self.pool.acquire() as connection:
resp = await connection.fetch(req)
users = [(discord.utils.get(guild.members,
id=int(x["row"][0])), x["row"][1])
for x in resp
if (len(x["row"]) == 2) and (x["row"][1] is not None)]
users = [x for x in users if x[0]]
users.sort(key=lambda x: -float(x[1]))
currency = await self.bot.di.get_currency(guild)
baltop = "\n".join(f"<li> {y[0]} {y[1]} {currency}</li>"
for y in users[:11])
characters = "\n".join(
f"<li>{name}</li>"
for name, obj in guild_data["characters"].items()
if obj[2] == str(medata["id"]))
hubbutton = """
<button>
<a href="/hub?token={token}">
Return to Guilds
</a>
</button>
""".format(token=token)
html = html.format(
start_money=start,
my_stats=stats,
my_inventory=inventory,
baltop=baltop,
my_characters=characters,
my_guild="Guild: " + str(user_data["guild"]),
my_box=None,
salaries=None,
items=None,
other_characters=None,
guilds=None,
shop=None,
market=None,
lotteries=None,
hubbutton=hubbutton,
guildname=guild.name,
)
resp = web.Response(body=html.encode())
resp.headers['content-type'] = 'text/html'
return resp
async def event_stream(request):
user_agent = request.headers.get("User-Agent")
username = user_agent[user_agent.find("user:"******"users"]
seeks = request.app["seeks"]
sockets = request.app["lobbysockets"]
games = request.app["games"]
db = request.app["db"]
resp = web.StreamResponse()
resp.content_type = "text/plain"
await resp.prepare(request)
if username in users:
bot_player = users[username]
# After BOT lost connection it may have ongoing games
# We notify BOT and he can ask to create new game_streams
# to continue those games
for gameId in bot_player.game_queues:
if gameId in games and games[gameId].status == STARTED:
await bot_player.event_queue.put(games[gameId].game_start)
else:
bot_player = User(request.app, bot=True, username=username)
users[bot_player.username] = bot_player
doc = await db.user.find_one({"_id": username})
if doc is None:
result = await db.user.insert_one({
"_id": username,
"title": "BOT",
})
print("db insert user result %s" % repr(result.inserted_id))
bot_player.online = True
log.info("+++ BOT %s connected", bot_player.username)
pinger_task = asyncio.create_task(
bot_player.pinger(sockets, seeks, users, games))
# inform others
# TODO: do we need this at all?
await lobby_broadcast(sockets, get_seeks(seeks))
# send "challenge" and "gameStart" events from event_queue to the BOT
while bot_player.online:
answer = await bot_player.event_queue.get()
try:
bot_player.event_queue.task_done()
except ValueError:
log.error(
"task_done() called more times than there were items placed in the queue in bot_api.py event_stream()"
)
try:
if request.protocol.transport.is_closing():
log.error(
"BOT %s request.protocol.transport.is_closing() == True ...",
username,
)
break
else:
await resp.write(answer.encode("utf-8"))
await resp.drain()
except Exception:
log.error("BOT %s event_stream is broken...", username)
break
pinger_task.cancel()
await bot_player.clear_seeks(force=True)
return resp
async def execute_parsed(
self, request: Request, command: str,
parsed: List[ParsedCommandLine]) -> StreamResponse:
# make sure, all requirements are fulfilled
not_met_requirements = [
not_met for line in parsed for not_met in line.unmet_requirements
]
# what is the accepted content type
# only required for multipart requests
boundary = "----cli"
mp_response = web.StreamResponse(
status=200,
reason="OK",
headers={"Content-Type": f"multipart/mixed;boundary={boundary}"})
async def list_or_gen(
current: ParsedCommandLine) -> Tuple[Optional[int], Stream]:
maybe_count, out_gen = await current.execute()
if (request.headers.get("accept") == "text/plain"
and current.executable_commands and
not isinstance(current.executable_commands[-1].command,
(OutputTransformer, PreserveOutputFormat))):
out_gen = await ListCommand(self.cli.dependencies
).parse(ctx=current.ctx
).flow(out_gen)
return maybe_count, out_gen
if not_met_requirements:
requirements = [
req for line in parsed for cmd in line.executable_commands
for req in cmd.action.required
]
data = {
"command": command,
"env": dict(request.query),
"required": to_json(requirements)
}
return web.json_response(data, status=424)
elif len(parsed) == 1:
first_result = parsed[0]
count, generator = await list_or_gen(first_result)
# flat the results from 0 or 1
async with generator.stream() as streamer:
gen = await force_gen(streamer)
if first_result.produces.json:
return await self.stream_response_from_gen(
request, gen, count)
elif first_result.produces.file_path:
await mp_response.prepare(request)
await Api.multi_file_response(first_result, gen, boundary,
mp_response)
return mp_response
else:
raise AttributeError(
f"Can not handle type: {first_result.produces}")
elif len(parsed) > 1:
await mp_response.prepare(request)
for single in parsed:
count, generator = await list_or_gen(single)
async with generator.stream() as streamer:
gen = await force_gen(streamer)
if single.produces.json:
with MultipartWriter(repr(single.produces),
boundary) as mp:
content_type, result_stream = await result_binary_gen(
request, gen)
mp.append_payload(
AsyncIterablePayload(result_stream,
content_type=content_type,
headers=single.envelope))
await mp.write(mp_response, close_boundary=True)
elif single.produces.file_path:
await Api.multi_file_response(single, gen, boundary,
mp_response)
else:
raise AttributeError(
f"Can not handle type: {single.produces}")
await mp_response.write_eof()
return mp_response
else:
raise AttributeError("No command could be parsed!")
async def prepare(self, request):
self._response = web.StreamResponse(headers={'Content-Type': 'text/plain'})
await self._response.prepare(self._request)
def func(request):
response = web.StreamResponse(status=200)
response.content_type = 'text/html'
response.start(request)
response.write(b'text')
return response
def function146(arg579):
var738 = arg579.clone()
var1805 = web.StreamResponse()
yield from var1805.prepare(var738)
return var1805
async def handler(request):
resp = web.StreamResponse(headers={'content-length': '100'})
await resp.prepare(request)
await resp.drain()
await asyncio.sleep(0.1, loop=request.app.loop)
return resp
def handler(request):
resp = web.StreamResponse()
yield from resp.prepare(request)
request.transport.close()
resp.write(b'answer' * 1000)
return resp
async def get(self, request):
"""Provide a streaming interface for the event bus."""
if not request["hass_user"].is_admin:
raise Unauthorized()
hass = request.app["hass"]
stop_obj = object()
to_write = asyncio.Queue()
restrict = request.query.get("restrict")
if restrict:
restrict = restrict.split(",") + [EVENT_HOMEASSISTANT_STOP]
async def forward_events(event):
"""Forward events to the open request."""
if event.event_type == EVENT_TIME_CHANGED:
return
if restrict and event.event_type not in restrict:
return
_LOGGER.debug("STREAM %s FORWARDING %s", id(stop_obj), event)
if event.event_type == EVENT_HOMEASSISTANT_STOP:
data = stop_obj
else:
data = json.dumps(event, cls=JSONEncoder)
await to_write.put(data)
response = web.StreamResponse()
response.content_type = "text/event-stream"
await response.prepare(request)
unsub_stream = hass.bus.async_listen(MATCH_ALL, forward_events)
try:
_LOGGER.debug("STREAM %s ATTACHED", id(stop_obj))
# Fire off one message so browsers fire open event right away
await to_write.put(STREAM_PING_PAYLOAD)
while True:
try:
with async_timeout.timeout(STREAM_PING_INTERVAL):
payload = await to_write.get()
if payload is stop_obj:
break
msg = f"data: {payload}\n\n"
_LOGGER.debug("STREAM %s WRITING %s", id(stop_obj), msg.strip())
await response.write(msg.encode("UTF-8"))
except asyncio.TimeoutError:
await to_write.put(STREAM_PING_PAYLOAD)
except asyncio.CancelledError:
_LOGGER.debug("STREAM %s ABORT", id(stop_obj))
finally:
_LOGGER.debug("STREAM %s RESPONSE CLOSED", id(stop_obj))
unsub_stream()
return response
def handler(request):
resp = web.StreamResponse()
yield from resp.prepare(request)
yield from asyncio.sleep(0.1, loop=loop)
return resp
def handler(request):
response = web.StreamResponse(status=200)
response.content_type = 'text/html'
yield from response.prepare(request)
response.write(b'text')
return response
def handler(request):
cloned = request.clone()
resp = web.StreamResponse()
yield from resp.prepare(cloned)
return resp
async def handle_delete(request):
await request.content.read()
response = web.StreamResponse(status=200,
headers={'connection': 'keep-alive'})
await response.prepare(request)
return web.Response(status=200)
def get(self, request):
"""Provide a streaming interface for the event bus."""
# pylint: disable=no-self-use
hass = request.app['hass']
stop_obj = object()
to_write = asyncio.Queue(loop=hass.loop)
restrict = request.query.get('restrict')
if restrict:
restrict = restrict.split(',') + [EVENT_HOMEASSISTANT_STOP]
@asyncio.coroutine
def forward_events(event):
"""Forward events to the open request."""
if event.event_type == EVENT_TIME_CHANGED:
return
if restrict and event.event_type not in restrict:
return
_LOGGER.debug('STREAM %s FORWARDING %s', id(stop_obj), event)
if event.event_type == EVENT_HOMEASSISTANT_STOP:
data = stop_obj
else:
data = json.dumps(event, cls=rem.JSONEncoder)
yield from to_write.put(data)
response = web.StreamResponse()
response.content_type = 'text/event-stream'
yield from response.prepare(request)
unsub_stream = hass.bus.async_listen(MATCH_ALL, forward_events)
try:
_LOGGER.debug('STREAM %s ATTACHED', id(stop_obj))
# Fire off one message so browsers fire open event right away
yield from to_write.put(STREAM_PING_PAYLOAD)
while True:
try:
with async_timeout.timeout(STREAM_PING_INTERVAL,
loop=hass.loop):
payload = yield from to_write.get()
if payload is stop_obj:
break
msg = "data: {}\n\n".format(payload)
_LOGGER.debug('STREAM %s WRITING %s', id(stop_obj),
msg.strip())
response.write(msg.encode("UTF-8"))
yield from response.drain()
except asyncio.TimeoutError:
yield from to_write.put(STREAM_PING_PAYLOAD)
except asyncio.CancelledError:
_LOGGER.debug('STREAM %s ABORT', id(stop_obj))
finally:
_LOGGER.debug('STREAM %s RESPONSE CLOSED', id(stop_obj))
unsub_stream()
async def do(self, http_method, req, resp, **kwargs):
assert http_method in ('delete', 'get', 'post', 'put')
methodname = getattr(self, http_method)
method = self.rest._methods[methodname]
"""
Arguments for a method can be grabbed from an override method in
the form of "get_{get,post,put,delete}_args", e.g.:
def get_post_args(self, req, resp, **kwargs):
return [await req.json(), True, False]
"""
get_method_args = getattr(self, 'get_{}_args'.format(http_method),
None)
if get_method_args is not None:
method_args = get_method_args(req, resp, **kwargs)
else:
method_args = []
if http_method == 'get' and method['filterable']:
if self.parent and 'id' in kwargs:
filterid = kwargs['id']
if filterid.isdigit():
filterid = int(filterid)
method_args = [[('id', '=', filterid)], {'get': True}]
else:
method_args = self._filterable_args(req)
if not method_args:
# RFC 7231 specifies that a GET request can accept a payload body
# This means that all the http methods now ( delete, get, post, put ) accept a payload body
try:
text = await req.text()
if not text:
method_args = []
else:
data = await req.json()
params = self.__method_params.get(methodname)
if not params and http_method in (
'get', 'delete') and not data:
# This will happen when the request body contains empty dict "{}"
# Keeping compatibility with how we used to accept the above case, this
# makes sure that existing client implementations are not affected
method_args = []
elif not params or len(params) == 1:
method_args = [data]
else:
if not isinstance(data, dict):
resp.set_status(400)
resp.body = json.dumps({
'message':
'Endpoint accepts multiple params, object/dict expected.',
})
return resp
method_args = []
for p, options in sorted(
params.items(),
key=lambda x: x[1]['order']):
if p not in data and options['required']:
resp.set_status(400)
resp.body = json.dumps({
'message':
f'{p} attribute expected.',
})
return resp
elif p in data:
method_args.append(data.pop(p))
if data:
resp.set_status(400)
resp.body = json.dumps({
'message':
f'The following attributes are not expected: {", ".join(data.keys())}',
})
return resp
except Exception as e:
resp.set_status(400)
resp.body = json.dumps({
'message': str(e),
})
return resp
"""
If the method is marked `item_method` then the first argument
must be the item id (from url param)
"""
if method.get('item_method') is True:
method_args.insert(0, kwargs['id'])
if method.get('pass_application'):
method_kwargs = {
'app':
Application(req.headers.get('X-Real-Remote-Addr'),
req.headers.get('X-Real-Remote-Port'))
}
else:
method_kwargs = {}
download_pipe = None
if method['downloadable']:
download_pipe = self.middleware.pipe()
method_kwargs['pipes'] = Pipes(output=download_pipe)
try:
result = await self.middleware.call(methodname, *method_args,
**method_kwargs)
except CallError as e:
resp = web.Response(status=422)
result = {
'message': e.errmsg,
'errno': e.errno,
}
except (SchemaError, ValidationError, ValidationErrors) as e:
if isinstance(e, (SchemaError, ValidationError)):
e = [(e.attribute, e.errmsg, e.errno)]
result = defaultdict(list)
for attr, errmsg, errno in e:
result[attr].append({
'message': errmsg,
'errno': errno,
})
resp = web.Response(status=422)
except Exception as e:
adapted = adapt_exception(e)
if adapted:
resp = web.Response(status=422)
result = {
'message': adapted.errmsg,
'errno': adapted.errno,
}
else:
if isinstance(e, (MatchNotFound, )):
resp = web.Response(status=404)
result = {
'message': str(e),
}
else:
resp = web.Response(status=500)
result = {
'message': str(e),
'traceback': ''.join(traceback.format_exc()),
}
if download_pipe is not None:
resp = web.StreamResponse(status=200,
reason='OK',
headers={
'Content-Type':
'application/octet-stream',
'Transfer-Encoding': 'chunked',
})
await resp.prepare(req)
loop = asyncio.get_event_loop()
def do_copy():
while True:
read = download_pipe.r.read(1048576)
if read == b'':
break
asyncio.run_coroutine_threadsafe(resp.write(read),
loop=loop).result()
await self.middleware.run_in_thread(do_copy)
await resp.drain()
return resp
if isinstance(result, types.GeneratorType):
result = list(result)
elif isinstance(result, types.AsyncGeneratorType):
result = [i async for i in result]
elif isinstance(result, Job):
result = result.id
resp.text = json.dumps(result, indent=True)
return resp
async def download(self, request):
path = request.path.split('/')
if not request.path[-1].isdigit():
resp = web.Response()
resp.set_status(404)
return resp
job_id = int(path[-1])
qs = urllib.parse.parse_qs(request.query_string)
denied = False
filename = None
if 'auth_token' not in qs:
denied = True
else:
auth_token = qs.get('auth_token')[0]
token = await self.middleware.call('auth.get_token', auth_token)
if not token:
denied = True
else:
if token['attributes'].get('job') != job_id:
denied = True
else:
filename = token['attributes'].get('filename')
if denied:
resp = web.Response()
resp.set_status(401)
return resp
job = self.middleware.jobs.get(job_id)
if not job:
resp = web.Response()
resp.set_status(404)
return resp
if job_id not in self.jobs:
resp = web.Response()
resp.set_status(410)
return resp
resp = web.StreamResponse(status=200, reason='OK', headers={
'Content-Type': 'application/octet-stream',
'Content-Disposition': f'attachment; filename="{filename}"',
'Transfer-Encoding': 'chunked',
})
await resp.prepare(request)
def do_copy():
while True:
read = job.pipes.output.r.read(1048576)
if read == b'':
break
asyncio.run_coroutine_threadsafe(resp.write(read), loop=self.loop).result()
try:
await self.middleware.run_in_thread(do_copy)
finally:
await self._cleanup_job(job_id)
await resp.drain()
return resp
async def serve_vgm(self, request):
vgm_data = await request.read()
digest = hashlib.sha256(vgm_data).hexdigest()[:16]
self._logger.info("web: %s: submitted by %s", digest, request.remote)
try:
if len(vgm_data) < 0x80:
raise ValueError("File is too short to be valid")
try:
vgm_stream = io.BytesIO(vgm_data)
if not vgm_data.startswith(b"Vgm "):
vgm_stream = gzip.GzipFile(fileobj=vgm_stream)
vgm_reader = VGMStreamReader(vgm_stream)
except OSError:
raise ValueError("File is not in VGM or VGZ format")
self._logger.info("web: %s: VGM has commands for %s", digest,
", ".join(vgm_reader.chips()))
if len(vgm_reader.chips()) != 1:
raise ValueError(
"VGM file contains commands for more than one chip")
clock_rate = vgm_reader.ym3526_clk or vgm_reader.ym3812_clk
if clock_rate == 0:
raise ValueError(
"VGM file does not contain commands for YM3526 or YM3812")
if clock_rate & 0xc0000000:
raise ValueError(
"VGM file uses unsupported chip configuration")
self._logger.info("web: %s: VGM is looped for %.2f/%.2f s", digest,
vgm_reader.loop_seconds,
vgm_reader.total_seconds)
vgm_player = YamahaVGMStreamPlayer(vgm_reader, self._opl_iface,
clock_rate)
except ValueError as e:
self._logger.warning("web: %s: broken upload: %s", digest, str(e))
return web.Response(status=400,
text=str(e),
content_type="text/plain")
input_rate = 1 / vgm_player.sample_time
preferred_rate = int(request.headers["X-Preferred-Sample-Rate"])
resample, output_rate = self._make_resampler(input_rate,
preferred_rate)
self._logger.info(
"web: %s: sample rate: input %d, preferred %d, output %d", digest,
input_rate, preferred_rate, output_rate)
async with self._lock:
self._logger.info("web: %s: start streaming", digest)
await self._opl_iface.reset()
input_queue = asyncio.Queue()
resample_queue = asyncio.Queue()
resample_fut = asyncio.ensure_future(
resample(input_queue, resample_queue))
record_fut = asyncio.ensure_future(vgm_player.record(input_queue))
play_fut = asyncio.ensure_future(vgm_player.play(disable=False))
try:
response = web.StreamResponse()
response.content_type = "text/plain"
response.headers["X-Sample-Rate"] = str(output_rate)
total_samples = int(vgm_reader.total_seconds * output_rate)
response.headers["X-Total-Samples"] = str(total_samples)
if vgm_reader.loop_samples in (0, vgm_reader.total_samples):
# Either 0 or the entire VGM here means we'll loop the complete track.
loop_skip_to = 0
else:
loop_skip_to = int(
(vgm_reader.total_seconds - vgm_reader.loop_seconds) *
output_rate)
response.headers["X-Loop-Skip-To"] = str(loop_skip_to)
response.enable_chunked_encoding()
await response.prepare(request)
TRANSPORT_SIZE = 3072
output_buffer = bytearray()
while True:
if not resample_fut.done() or not resample_queue.empty():
while len(output_buffer) < TRANSPORT_SIZE:
output_chunk = await resample_queue.get()
output_buffer += output_chunk
if not output_chunk:
break
transport_chunk = output_buffer[:TRANSPORT_SIZE]
while len(transport_chunk) < TRANSPORT_SIZE:
# Pad last transport chunk with silence
transport_chunk += struct.pack("<H", 32768)
output_buffer = output_buffer[TRANSPORT_SIZE:]
await response.write(base64.b64encode(transport_chunk))
if resample_fut.done() and not output_buffer:
break
for fut in [play_fut, record_fut, resample_fut]:
await fut
await response.write_eof()
self._logger.info("web: %s: done streaming", digest)
except asyncio.CancelledError:
self._logger.info("web: %s: cancel streaming", digest)
for fut in [play_fut, record_fut, resample_fut]:
if not fut.done():
fut.cancel()
raise
return response
async def download(self, job: StorageUWSJob, request: aiohttp.web.Request):
"""Download files from the NGAS server"""
path_tree = job.transfer.target.path
if job.transfer.target.node_type == NodeType.ContainerNode:
# Download the files from NGAS to a directory,
# tar the directory and send to client
# Should we implement a streaming creation of the tarfile?
# Make a directory in staging area
path_tree = job.transfer.target.path
stage_dir = os.path.normpath(f'{self.staging_dir}/{uuid.uuid4()}')
stage_path = os.path.normpath(f'{stage_dir}/{path_tree}')
# make the stage directory
await mkdir(stage_dir)
# Now loop over each current node in NGAS and fetch files
current_node = job.transfer.target
# Current node
if job.transfer.view != View('ivo://ivoa.net/vospace/core#tar'):
return web.Response(status=400,
text=f'Unsupported Container View. '
f'View: {job.transfer.view}')
# Now loop over each node and download to file
for node in current_node.walk(current_node):
node_path = os.path.normpath(f'{stage_dir}/{node.path}')
if node.node_type == NodeType.ContainerNode:
# Make a directory in the node path
await mkdir(node_path)
else:
# Make the local filename
filename_local = node_path
# Make the NGAS filename
filename_ngas = f'{os.path.basename(node_path)}_{node.id}'
# Fetch a file from NGAS
await recv_file_from_ngas(self.ngas_session,
self.ngas_hostname,
self.ngas_port, filename_ngas,
filename_local)
# Tar file
tar_file = os.path.normpath(
f'{stage_dir}/{uuid.uuid4()}/{os.path.basename(path_tree)}.tar'
)
try:
# Tarring up the sources, I might need to ask the database for all
loop = asyncio.get_event_loop()
await loop.run_in_executor(self.process_executor, tar,
stage_path, tar_file,
os.path.basename(path_tree))
return await send_file(request, os.path.basename(tar_file),
tar_file)
finally:
with suppress(Exception):
await asyncio.shield(rmtree(os.path.dirname(tar_file)))
with suppress(Exception):
await asyncio.shield(rmtree(os.path.dirname(stage_dir)))
else:
# Get the UUID on the node from the database
id = job.transfer.target.id
# Filename to be used with the NGAS object store
base_name = os.path.basename(path_tree)
filename_ngas = f'{base_name}_{id}'
# URL for retrieval from NGAS
url_ngas = f'http://{self.ngas_hostname}:{self.ngas_port}/RETRIEVE'
# Make up the filename for retrieval from NGAS
params = {"file_id": filename_ngas}
# Connect to NGAS
resp_ngas = await self.ngas_session.get(url_ngas, params=params)
# Rudimentry error checking on the NGAS connection
if resp_ngas.status != 200:
raise aiohttp.web.HTTPServerError(
reason="Error in connecting to NGAS server")
# Otherwise create the client
resp_client = web.StreamResponse()
# Update the headers
resp_client.headers.update(resp_ngas.headers)
# Change the filename?
resp_client.headers[
'Content-Disposition'] = f'attachment; filename=\"{base_name}\"'
# Prepare the connection
await resp_client.prepare(request)
# Read from source and and write destination in buffers
async for chunk in resp_ngas.content.iter_chunked(
io.DEFAULT_BUFFER_SIZE):
if chunk:
await resp_client.write(chunk)
# Finish the stream
await resp_client.write_eof()
return (resp_client)
async def handler(request):
resp = web.StreamResponse()
await resp.prepare(request)
await resp.drain()
await asyncio.sleep(10, loop=loop)
return resp
async def event_stream(request):
auth = request.headers.get("Authorization")
if auth is None:
return web.HTTPForbidden()
token = auth[auth.find("Bearer") + 7:]
if token not in BOT_TOKENS:
log.error("BOT account auth with token %s failed" % token)
return web.HTTPForbidden()
user_agent = request.headers.get("User-Agent")
username = user_agent[user_agent.find("user:"******"users"]
seeks = request.app["seeks"]
sockets = request.app["websockets"]
games = request.app["games"]
db = request.app["db"]
resp = web.StreamResponse()
resp.content_type = "text/plain"
await resp.prepare(request)
if username in users:
bot_player = users[username]
# After BOT lost connection it may have ongoing games
# We notify BOT and he can ask to create new game_streams
# to continue those games
for gameId in bot_player.game_queues:
if gameId in games and games[gameId].status == STARTED:
await bot_player.event_queue.put(games[gameId].game_start)
else:
bot_player = User(request.app, bot=True, username=username)
users[bot_player.username] = bot_player
doc = await db.user.find_one({"_id": username})
if doc is None:
result = await db.user.insert_one({
"_id": username,
"first_name": None,
"last_name": None,
"country": None,
"title": "BOT",
})
print("db insert user result %s" % repr(result.inserted_id))
bot_player.bot_online = True
log.info("+++ BOT %s connected" % bot_player.username)
loop = asyncio.get_event_loop()
pinger_task = loop.create_task(bot_player.pinger(sockets, seeks, users, games))
# inform others
# TODO: do we need this at all?
await lobby_broadcast(sockets, get_seeks(seeks))
# send "challenge" and "gameStart" events from event_queue to the BOT
while bot_player.online():
answer = await bot_player.event_queue.get()
try:
if request.protocol.transport.is_closing():
log.error("BOT %s request.protocol.transport.is_closing() == True ..." % username)
break
else:
await resp.write(answer.encode("utf-8"))
await resp.drain()
except Exception:
log.error("BOT %s event_stream is broken..." % username)
break
pinger_task.cancel()
await bot_player.clear_seeks(sockets, seeks)
return resp
async def download(self, request):
path = request.path.split('/')
if not request.path[-1].isdigit():
resp = web.Response()
resp.set_status(404)
return resp
job_id = int(path[-1])
jobs = self.middleware.jobs.all()
job = jobs.get(job_id)
if not job:
resp = web.Response()
resp.set_status(404)
return resp
qs = urllib.parse.parse_qs(request.query_string)
denied = False
filename = None
if 'auth_token' not in qs:
denied = True
else:
auth_token = qs.get('auth_token')[0]
token = await self.middleware.call('auth.get_token', auth_token)
if not token:
denied = True
else:
if (token['attributes'] or {}).get('job') != job_id:
denied = True
else:
filename = token['attributes'].get('filename')
if denied:
resp = web.Response()
resp.set_status(401)
return resp
resp = web.StreamResponse(status=200,
reason='OK',
headers={
'Content-Type':
'application/octet-stream',
'Content-Disposition':
f'attachment; filename="{filename}"',
'Transfer-Encoding': 'chunked',
})
await resp.prepare(request)
f = None
try:
def read_write():
f = os.fdopen(job.read_fd, 'rb')
while True:
read = f.read(1024)
if read == b'':
break
resp.write(read)
#await web.drain()
await self.middleware.threaded(read_write)
await resp.drain()
finally:
if f:
f.close()
return resp
async def video(request):
videoId = request.match_info['videoId']
#random.seed(hash(videoId))
logger.append('video', request.identity, videoId)
response = web.StreamResponse()
response.content_type = 'text/html'
response.set_cookie(
'identity', request.identity) # needs to be set before sending data
await response.prepare(request)
await response.write(template().encode('utf8'))
#await response.write('<center id="player"></center>\n'.encode('utf8'))
# load video info + player
#async def get_player():
# async for item in youtube.video([videoId], 50):
# # inject content when player get available
# await response.write(('<script>document.getElementById("player").innerHTML = \'%s\';</script>\n' % item['player']['embedHtml']).encode('utf8'))
#task1 = asyncio.create_task(get_player())
await response.write((
'''<center><iframe id="ytplayer" type="text/html" width="640" height="390" src="https://www.youtube-nocookie.com/embed/'''
+ videoId +
'''?rel=0&showinfo=0&iv_load_policy=0" frameborder="0" allow="autoplay" allowfullscreen></iframe></center>'''
).encode('utf8'))
#await response.write(('''
# <center><iframe id="ytplayer" type="text/html" width="640" height="360"
# src="https://www.youtube.com/embed/''' + videoId + '''?autoplay=1"
# frameborder="0"></iframe></center>
#''').encode('utf8'))
# select random video and check that it exists
async def get_random():
while True:
value = random.random()
if value < .5:
chosen = hint_ids.choice()
else:
chosen = random_ids.choice()
if await youtube.exists(chosen):
return chosen
# load recommendations
async def get_related():
# iterator over 100 random youtube videos
random_selection = asyncio.as_completed(
[asyncio.create_task(get_random()) for _ in range(100)])
await response.write('<table id="thumbnails">\n<tr>'.encode('utf8'))
num = 0
async for item in youtube.related(videoId, 50):
if num > 0 and num % 3 == 0:
# add separator
await response.write(
'<td> </td>'.encode('utf8'))
# add 3 videos from the random selection
for _ in range(3):
try:
result = await next(random_selection)
except StopIteration:
# feed more random videos as needed
random_selection = asyncio.as_completed([
asyncio.create_task(get_random())
for _ in range(100)
])
result = await next(random_selection)
await response.write((
'<td class="thumbnail"><a href="/video/{id}"><img src="https://i.ytimg.com/vi/{id}/mqdefault.jpg"></a></td>'
.format(id=result)).encode('utf8'))
await response.write('</tr>\n<tr>'.encode('utf8'))
await response.write((
'<td class="thumbnail"><a href="/video/{id}"><img src="https://i.ytimg.com/vi/{id}/mqdefault.jpg"></a></td>'
.format(id=item['id']['videoId'])).encode('utf8'))
num += 1
task2 = asyncio.create_task(get_related())
# perform the two tasks in parallel
#await task1
await task2
# trim last row
await response.write((
'</table><center style="color: gray">' + str(datetime.now()) + ' ' +
request.identity +
'</center>\n<script> document.querySelector("tbody tr:last-child").remove(); </script>\n'
).encode('utf8'))
# load random stuff
return response