def go(dirname, filename):
ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
ssl_ctx.load_cert_chain(
os.path.join(dirname, 'sample.crt'),
os.path.join(dirname, 'sample.key')
)
app, _, url = yield from self.create_server(
'GET', '/static/' + filename, ssl_ctx=ssl_ctx
)
app.router.add_static('/static', dirname)
conn = TCPConnector(verify_ssl=False, loop=self.loop)
session = ClientSession(connector=conn)
resp = yield from session.request('GET', url)
self.assertEqual(200, resp.status)
txt = yield from resp.text()
self.assertEqual('file content', txt.rstrip())
ct = resp.headers['CONTENT-TYPE']
self.assertEqual('application/octet-stream', ct)
self.assertEqual(resp.headers.get('CONTENT-ENCODING'), None)
resp.close()
resp = yield from session.request('GET', url + 'fake')
self.assertEqual(404, resp.status)
resp.close()
resp = yield from session.request('GET', url + '/../../')
self.assertEqual(404, resp.status)
resp.close()
def github_action(action, plugin, config, sort_by='updated'):
url = API_URL.format(
api=API_URL,
user=config['user'],
repo=config['repository'],
action=action,
)
query = {
'sort': sort_by,
'direction': 'desc',
'sha': config.get('branch', 'master')
}
headers = {'Accept': 'application/vnd.github.v3+json'}
etag = plugin.temp.get(action)
if etag is not None:
headers['If-None-Match'] = etag
session = ClientSession()
try:
resp = yield from asyncio.wait_for(
session.get(url, params=query, headers=headers),
timeout=5
)
try:
plugin.temp[action] = resp.headers.get('etag')
if resp.status != 200 or etag is None: # etag must be cached first
raise NothingChangeException(etag)
data = yield from resp.json()
finally:
resp.close()
finally:
session.close()
return data[0]
def go():
_, srv, url = yield from self.create_server(None, '/', None)
client = ClientSession(loop=self.loop)
resp = yield from client.get(url)
self.assertEqual(404, resp.status)
yield from resp.release()
client.close()
def go():
_, srv, url = yield from self.create_server('GET', '/', handler)
client = ClientSession(loop=self.loop)
resp = yield from client.get(url)
self.assertEqual(200, resp.status)
data = yield from resp.read()
self.assertEqual(b'xyz', data)
yield from resp.release()
def go():
_, srv, url = yield from self.create_server('GET', '/', handler)
client = ClientSession(loop=self.loop)
resp = yield from client.get(url)
self.assertEqual(200, resp.status)
data = yield from resp.read()
self.assertEqual(b'mydata', data)
self.assertEqual(resp.headers.get('CONTENT-ENCODING'), 'deflate')
yield from resp.release()
client.close()
def get_html(url):
session = ClientSession()
try:
resp = yield from session.get(url)
try:
raw_data = yield from resp.read()
data = raw_data.decode('utf-8', errors='xmlcharrefreplace')
return fromstring_to_html(data)
finally:
resp.close()
finally:
session.close()
def __init__(self, configuration):
self._limit_sleep_time_coefficient = configuration \
.instagram_limit_sleep_time_coefficient
self._limit_sleep_time_min = configuration \
.instagram_limit_sleep_time_min
self._success_sleep_time_coefficient = configuration \
.instagram_success_sleep_time_coefficient
self._success_sleep_time_max = configuration \
.instagram_success_sleep_time_max
self._success_sleep_time_min = configuration \
.instagram_success_sleep_time_min
self._limit_sleep_time = self._limit_sleep_time_min
self._success_sleep_time = self._success_sleep_time_max
self._username = configuration.instagram_username
self._password = configuration.instagram_password
self._referer = BASE_URL
self._session = ClientSession(
cookies={
'ig_pr': '1',
'ig_vw': '1920',
},
headers={
'User-Agent': USER_AGENT,
'X-Instagram-AJAX': '1',
'X-Requested-With': 'XMLHttpRequest',
},
)
loop = asyncio.get_event_loop()
loop.run_until_complete(self._do_login())
async def query_imdb(*, imdb_id: Optional[int],
year: int,
session: ClientSession) -> Dict[str, Any]:
params = dict(i=f'tt{imdb_id:0>{IMDB_ID_LENGTH}}',
y=year,
plot='full',
tomatoes='true',
r='json')
while True:
attempt_num = 0
async with session.get(IMDB_API_URL, params=params) as response:
attempt_num += 1
if response.status == A_TIMEOUT_OCCURRED:
logger.debug(f'Attempt #{attempt_num} failed: '
f'server "{IMDB_API_URL}" answered with '
f'status code {A_TIMEOUT_OCCURRED}. '
f'Waiting {RETRY_INTERVAL_IN_SECONDS} second(s) '
'before next attempt.')
await sleep(RETRY_INTERVAL_IN_SECONDS)
continue
try:
response_json = await response.json()
except JSONDecodeError:
logger.exception('')
return dict()
return response_json
async def check(self, aio_client: aiohttp.ClientSession,
usernames: AbstractSet[str]) -> ni_abc.Status:
base_url = "https://bugs.python.org/user?@template=clacheck&github_names="
url = base_url + ','.join(usernames)
self.server.log("Checking CLA status: " + url)
async with aio_client.get(url) as response:
if response.status >= 300:
msg = f'unexpected response for {response.url!r}: {response.status}'
raise client.HTTPException(msg)
# Explicitly decode JSON as b.p.o doesn't set the content-type as
# `application/json`.
results = json.loads(await response.text())
self.server.log("Raw CLA status: " + str(results))
status_results = [results[k] for k in results.keys() if k in usernames]
self.server.log("Filtered CLA status: " + str(status_results))
if len(status_results) != len(usernames):
raise ValueError("# of usernames don't match # of results "
"({} != {})".format(len(usernames), len(status_results)))
elif any(x not in (True, False, None) for x in status_results):
raise TypeError("unexpected value in " + str(status_results))
if all(status_results):
return ni_abc.Status.signed
elif any(value is None for value in status_results):
return ni_abc.Status.username_not_found
else:
return ni_abc.Status.not_signed
async def fetch_with_sleep(session: aiohttp.ClientSession,
url: str,
sleep_time: int) -> Response:
async with session.get(url) as response:
print(f"request {url} with {sleep_time}s")
await sleep(sleep_time)
print(f"request {url} sleep done")
return response
async def _fetch_url(
url: str,
session: aiohttp.ClientSession,
timeout: float = 10.0) -> str:
with async_timeout.timeout(timeout):
async with session.get(url) as response:
response.raise_for_status()
return await response.text()
class Bot:
'''Base class of your bot.
You should implement the ``on_message`` function.
Text may be parsed using ``parse``
'''
def __init__(self):
self.conversations = {}
self.history = []
self._client = ClientSession()
def __enter__(self):
return self
def __exit__(self, ex_type, ex_value, ex_tb):
self._client.close()
def close(self):
self._client.close()
async def parse(self, text):
async with self._client.post(
LUIS_ENDPOINT,
text.encode('utf-8'),
) as resp:
data = await resp.json()
# TODO: Extract relevant information from data
return data
@abstractmethod
async def on_message(self, conversation, text):
pass
async def _handler(self, request):
return web.Response(
b"Not implemented",
)
async def _history(self, request):
return web.Response(
'\n'.join('<p>{!r}</p>'.format(h) for h in self.history).encode('utf-8')
)
async def afetch(session: ClientSession, url: str):
"""
Asynchronous fetch. Do a GET request, return text, properly shutdown
:param session: ClientSession object for aiohttp connection
:param url: The URL we want
:return:
"""
async with session.get(url) as response:
return await response.text()
class Downloader(Actor):
async def startup(self):
self.session = ClientSession(loop=self.loop)
@concurrent
async def download_content(self, url):
async with self.session.get(url) as response:
content = await response.read()
print('{}: {:.80}...'.format(url, content.decode()))
return len(content)
async def shutdown(self):
self.session.close()
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
await self.close()
async def process_partition(
loop: asyncio.BaseEventLoop,
results_queue: asyncio.Queue,
server_address: URL,
http: aiohttp.ClientSession,
partition: PointsPartition,
mission_template: Template,
mission_loader: str,
mission_name: str,
width: int,
scale: int,
) -> Awaitable[None]:
LOG.debug(
f"query range [{partition.start}:{partition.end}] on server "
f"{server_address}"
)
file_name = f"{mission_name}_{partition.start}_{partition.end}.mis"
missions_url = server_address / "missions"
mission_dir_url = missions_url / "heightmap"
mission_url = mission_dir_url / file_name
points = (
index_to_point(i, width, scale)
for i in range(partition.start, partition.end + 1)
)
mission = mission_template.render(
loader=mission_loader,
points=points,
)
data = FormData()
data.add_field(
'mission',
mission.encode(),
filename=file_name,
content_type='plain/text',
)
await http.post(mission_dir_url, data=data)
await http.post(mission_url / "load")
await http.post(missions_url / "current" / "begin")
async with http.get(server_address / "radar" / "stationary-objects") as response:
data = await response.json()
data = [
pack(HEIGHT_PACK_FORMAT, int(point['pos']['z']))
for point in data
]
data = b''.join(data)
await http.post(missions_url / "current" / "unload")
await http.delete(mission_url)
await results_queue.put((partition, data))
def __init__(self, config):
config = self.trafaret.check(config)
self.base_url = config['stream']
self.signature = HmacSha1Signature()
self.session = ClientSession()
self.oauth_client = oauth.Client(
client_key=config['api_key'],
client_secret=config['api_secret'],
resource_owner_key=config['access_token'],
resource_owner_secret=config['access_secret']
)
def __init__(self, url, dumper=None, loop=None):
self.url = url
self.dumper = dumper
if not loop:
loop = asyncio.get_event_loop()
if not self.dumper:
self.dumper = json.dumps
self.client = ClientSession(
loop=loop,
headers={'content-type': 'application/json'})
async def close_self_reaction_pr(number: int, session: aiohttp.ClientSession):
# PATCH /repos/:owner/:repo/pulls/:number
pr_url = github_url(
f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/pulls/{number}")
await session.patch(pr_url, headers=HEADERS, data=b'{"state": "closed"}')
# POST /repos/:owner/:repo/issues/:number/comments
comment_url = github_url(
f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/issues/{number}/comments")
async with session.post(comment_url, headers=HEADERS, data=json.dumps({"body": "\> Reacting to your own PR"})) as resp:
content = await resp.text()
print(f"RESPONSE: {resp.status}, CONTENT: {content}")
def __init__(self, app, protocol="http"):
self._app = app
self._loop = loop = app.loop
self.port = unused_port()
self._handler = handler = app.make_handler()
self._server = loop.run_until_complete(loop.create_server(
handler, '127.0.0.1', self.port
))
self._session = ClientSession(loop=self._loop)
self._root = "{}://127.0.0.1:{}".format(
protocol, self.port
)
self._closed = False
class StreamingClient:
filter_endpoint = "/1.1/statuses/filter.json"
sample_endpoint = "/1.1/statuses/sample.json"
predicates = ["follow", "locations", "track"]
keys = ['language', 'delimited', 'stall_warnings']
trafaret = t.Dict({
'api_key': t.String,
'api_secret': t.String,
'access_token': t.String,
'access_secret': t.String,
'stream': t.URL,
}).ignore_extra('*')
def __init__(self, config):
config = self.trafaret.check(config)
self.base_url = config['stream']
self.signature = HmacSha1Signature()
self.session = ClientSession()
self.oauth_client = oauth.Client(
client_key=config['api_key'],
client_secret=config['api_secret'],
resource_owner_key=config['access_token'],
resource_owner_secret=config['access_secret']
)
async def request(self, method, endpoint, params=None, headers=None):
url = urljoin(self.base_url, endpoint)
headers = (headers or {})
if method == 'POST':
headers['Content-type'] = 'application/x-www-form-urlencoded'
uri, signed_headers, body = self.oauth_client.sign(url, method, params, headers)
logger.debug("PREPARED: %s %s %s", uri, signed_headers, body)
resp = await self.session.request(method, uri, params=body, headers=signed_headers)
return resp
async def stream(self, queue, predicate, value, **kwargs):
params = dict(((predicate, value,),))
while True:
resp = await self.request('POST', self.filter_endpoint, params)
while True:
data = await resp.content.readline()
message = data.strip()
if message != b'':
message = json.loads(message.decode('utf-8'))
logger.debug("Got something %s", message['id'])
await queue.put(message)
await asyncio.sleep(10)
def __del__(self):
asyncio.ensure_future(self.session.close())
async def bfetch(session: ClientSession, url: str):
"""
Asynchronous binary fetch. Do a GET request, return binary data, properly shutdown
:param session: ClientSession object for aiohttp connection
:param url: The URL we want
:return:
"""
async with session.get(url) as response:
if response.status >= 400:
LOGGER.error("Got status %s for GET %s", response.status, url)
response.raise_for_status()
return await response.read()
class Github:
_BASE_URL = 'https://api.github.com'
def __init__(self, username, password, timeout=10):
self._loop = asyncio.get_event_loop()
self._session = ClientSession(loop=self._loop, auth=BasicAuth(username, password))
self._timeout = timeout
def close(self):
self._session.close()
async def fetch(self, url, params):
with Timeout(self._timeout):
async with self._session.get('{}{}'.format(self._BASE_URL, url), params=params) as response:
return await response.json()
async def search_repositories(self, language, pushed, sort, order):
q = 'language:{}'.format(language)
if pushed:
q = '{} pushed:>={}'.format(q, pushed)
params = {'q': q, 'sort': sort, 'order': order}
return await self.fetch('/search/repositories', params=params)
async def get_imdb_id(article_title: str, *,
session: ClientSession
) -> Optional[int]:
params = dict(action='expandtemplates',
text='{{IMDb title}}',
prop='wikitext',
title=article_title,
format='json')
async with session.get(WIKIPEDIA_API_URL,
params=params) as response:
response_json = await response.json()
templates = response_json['expandtemplates']
imdb_link = templates.get('wikitext', '')
search_res = IMDB_ID_RE.search(imdb_link)
if search_res is None:
return None
return int(search_res.group(0))
class RestClient():
def __init__(self):
self.client = ClientSession()
def close(self):
self.client.close()
@asyncio.coroutine
def post(self, url, headers={}):
if 'content-type' not in headers:
headers['content-type'] = 'application/json'
resp = yield from self.client.post(url, headers=headers)
data = yield from resp.read()
yield from resp.release()
return data.decode('utf-8')
@asyncio.coroutine
def get(self, url, headers={}, to_json=False):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
if to_json:
data = json.loads(data.decode('utf-8'))
return data
@asyncio.coroutine
def put(self, url, headers={}):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
@asyncio.coroutine
def delete(self, url):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
@asyncio.coroutine
def patch(self, url):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
async def create_line_to_position_map(
self, client_session: aiohttp.ClientSession
) -> MutableMapping[str, Dict[int, int]]:
result = defaultdict(dict) # type: MutableMapping[str, Dict[int, int]]
current_file = ''
position = -1
right_line_number = -1
headers = {
'Accept': 'application/vnd.github.diff',
}
url = ('{api_url}/repos/'
'{organization}/{repo}/pulls/{pr}'.format(
api_url=GITHUB_API_URL,
organization=self.organization,
repo=self.repo,
pr=self.pr))
async with client_session.get(url, headers=headers) as response:
async for line in response.content:
line = line.decode()
file_match = FILE_START_REGEX.match(line)
if file_match:
current_file = file_match.groups()[0].strip()
right_line_number = -1
position = -1
continue
elif line.startswith(NEW_FILE_SECTION_START):
current_file = ''
if not current_file:
continue
position += 1
hunk_match = HUNK_REGEX.match(line)
if hunk_match:
right_line_number = int(hunk_match.groups()[0]) - 1
elif not line.startswith('-'):
right_line_number += 1
result[current_file][right_line_number] = position
return result
def create_line_to_position_map(
self, client_session: aiohttp.ClientSession
) -> MutableMapping[str, Dict[int, int]]:
result = defaultdict(dict) # type: MutableMapping[str, Dict[int, int]]
current_file = ''
position = -1
right_line_number = -1
headers = {
'Accept': 'application/vnd.github.diff',
}
url = ('https://api.github.com/repos/'
'{organization}/{repo}/pulls/{pr}'.format(
organization=self.organization,
repo=self.repo,
pr=self.pr))
response = yield from client_session.get(url, headers=headers)
content = yield from response.text()
for line in content.split('\n'):
file_match = FILE_START_REGEX.match(line)
if file_match:
current_file = file_match.groups()[0]
right_line_number = -1
position = -1
continue
elif line.startswith(NEW_FILE_SECTION_START):
current_file = ''
if not current_file:
continue
position += 1
hunk_match = HUNK_REGEX.match(line)
if hunk_match:
right_line_number = int(hunk_match.groups()[0]) - 1
elif not line.startswith('-'):
right_line_number += 1
result[current_file][right_line_number] = position
return result
async def call(session: aiohttp.ClientSession,
params: Dict[str, Any],
*,
identity: Optional[Dict[str, str]] = None,
check: bool = False
) -> Dict[str, Any]:
"""An asyncio coroutine which makes a HTTP request to the
Ninchat Call API using the third-party aiohttp package.
If check is set, raises a ninchat.call.APIError on "error" reply
event.
"""
data = lib.request_content(params, identity=identity)
async with session.post(lib.url, data=data, headers=lib.request_headers) as r:
if r.status != HTTPStatus.OK:
r.raise_for_status()
e = await r.json()
if check:
lib.check_event(e)
return e
async def get_url(
url: str, session: ClientSession, with_text=False, with_data=False, headers=None
):
"""Fetch url.
This function ensures that only one request is sent to a domain at one point in time and that the same url is not
queried more than once.
Parameters
----------
url : str
session: ClientSession
with_text: bool
with_data : bool
headers: dict
Returns
-------
RequestResult
"""
o = urlparse(url)
if len(o.netloc) == 0:
return RequestResult(exception=f"Could not parse URL: {url}")
async with domain_lock:
if o.netloc not in domain_locks:
domain_locks[o.netloc] = asyncio.Lock()
lock = domain_locks[o.netloc]
async with lock:
if url not in response_cache:
for _ in range(3):
try:
logging.debug(f"GET {url}")
async with session.request(
method="GET", url=url, ssl=False, headers=headers
) as response:
status = response.status
if with_text:
try:
text = await response.text()
except:
text = await response.read()
response_cache[url] = RequestResult(
status=status, text=text
)
elif with_data:
data = await response.read()
response_cache[url] = RequestResult(
status=status, text=data
)
else:
response_cache[url] = RequestResult(status=status)
except asyncio.TimeoutError:
response_cache[url] = RequestResult(exception=f"Timeout for: {url}")
except Exception as e:
logging.debug(f"Error for: {url} ({e})")
response_cache[url] = RequestResult(
exception=f"Exception {e} for: {url}"
)
if RequestResult.exception is None:
break
await asyncio.sleep(15)
else:
logging.debug(f"Cached {url}")
return response_cache[url]
async def update_token(self):
parsed_url = urlparse(self.settings.get("api_url"))
self.api_url = "{url.scheme}://{url.netloc}".format(url=parsed_url)
LOGGER.debug("API URL OYD %s", self.api_url)
client_id = self.settings.get("client_id")
client_secret = self.settings.get("client_secret")
grant_type = self.settings.get("grant_type", "client_credentials")
scope = self.settings.get("scope")
if self.api_url is None:
raise PDSError("Please configure the plugin, api_url is empty")
if client_id is None:
raise PDSError("Please configure the plugin, client_id is empty")
if client_secret is None:
raise PDSError(
"Please configure the plugin, client_secret is empty")
async with ClientSession() as session:
body = {
"client_id": client_id,
"client_secret": client_secret,
"grant_type": grant_type,
}
if scope is not None:
body["scope"] = scope
result = await session.post(
self.api_url + "/oauth/token",
json=body,
)
result = await unpack_response(result)
token = json.loads(result)
self.token = token
self.token_timestamp = time.time()
LOGGER.info("update token: %s", self.token)
"""
Download the usage policy
"""
url = f"{self.api_url}/api/meta/usage"
async with ClientSession() as session:
result = await session.get(
url,
headers={
"Authorization": "Bearer " + self.token["access_token"]
},
)
result = await unpack_response(result)
self.settings["usage_policy"] = result
LOGGER.debug("Usage policy %s", self.settings["usage_policy"])
"""
Upload usage_policy as oca_schema_chunk
"""
async with ClientSession() as session:
result = await session.post(
"https://governance.ownyourdata.eu/api/usage-policy/parse",
headers={
"Authorization": "Bearer " + self.token["access_token"]
},
json={"ttl": self.settings["usage_policy"]},
)
result = await unpack_response(result)
result = json.loads(result)
result, err = map_parsed_usage_policy(
result, cached_schema_to_map_against)
await self.save(
result,
{
"table":
"tda.oca_chunks.H5F2YgEbXpSZjcNqAYevfGPFXSWUV1d2PnVg2ubkkKb"
},
addition_meta={"missing": err},
)
async def start(self):
"""Start the outbound transport."""
self.client_session = ClientSession(cookie_jar=DummyCookieJar(), trust_env=True)
return self
async def get_connection(url):
s = ClientSession()
sessions.append(s)
conn = await s.get(url)
dot()
return conn
async def fetch_json(client_session: ClientSession, url) -> dict:
async with client_session.get(url) as response:
return await response.json()
async def get_bytes(url, headers=None):
async with ClientSession() as asyncSession:
async with asyncSession.get(url, headers=headers) as response:
a = await response.read()
return a
result = await resp.read()
print('order: {:d}, result: {:s}'.format(order, str(result)))
def request_page(order, session, loop):
co = _request_page(order, session, loop)
asyncio.run_coroutine_threadsafe(co, loop)
try:
loop = asyncio.get_event_loop()
loopThread = threading.Thread(target=keep_loop, args=(loop, ))
loopThread.setDaemon(True)
loopThread.start()
session = ClientSession(loop=loop)
for i in range(5):
loop.call_soon_threadsafe(request_page, i, session, loop)
## loop.call_soon()会卡住
## loop.call_soon(request_page, i, session, loop)
## join()时ctrl-c无法结束, 而while True可以.
## loopThread.join()
while True:
time.sleep(1)
except KeyboardInterrupt as e:
## 以下两行可以注释掉, 不会报错
asyncio.run_coroutine_threadsafe(session.close(), loop)
time.sleep(1)
def session(self) -> ClientSession:
if self._session is None:
self._session = ClientSession(loop=self.loop)
return self._session
def __init__(self):
trace_config = TraceConfig()
trace_config.on_request_start.append(self.on_request_start)
trace_config.on_request_end.append(self.on_request_end)
self.session = ClientSession(trace_configs=[trace_config])
self.logger = AppLogger()
async def fetch(session: aiohttp.ClientSession, url: Address) -> Json:
async with session.get(url) as response:
output = await response.json()
# turning this into an async iterator instead of just returning the json gives us more chances to switch tasks
return output
async def post_url(url: str, headers: dict = None) -> Tuple[ClientResponse, dict]:
headers = headers or get_headers()
async with ClientSession(headers=headers) as session:
async with timeout(10):
async with session.post(url) as resp:
return await _extract_response_and_json_from_request(resp)
async def fetch_data(data):
async with ClientSession() as session:
url_data = url.format(data)
async with session.get(url_data) as resp:
data = json.loads(await resp.text())
return data['data']
while True:
if len(healthy) > 90:
healthy.remove(random.choice(healthy))
random.shuffle(healthy)
tasks = []
for s in healthy:
url = f'{base}/api/health/main/{s}'
task = asyncio.ensure_future(bound_post(sem, url, session))
await asyncio.sleep(random.random() / 4)
tasks.append(task)
await asyncio.gather(*tasks)
async def run_demo():
while True:
shard = random.randint(0, shards - 1)
state = random.randint(0, 6)
url = f'{base}/api/status/main/{shard}/{state}'
del url
async def main(session):
await run_init(session)
await run_healthy(session)
loop = asyncio.get_event_loop()
with ClientSession() as session:
loop.run_until_complete(main(session))
from aiohttp import ClientSession, web
from aiohttp.web import HTTPBadRequest, HTTPUnsupportedMediaType
import asyncio
import logging
from lib.metrics import REQUEST_TIME
from lib.scoring import score
from lib.util import fetch
from prometheus_async.aio import time
import uvloop
logger = logging.getLogger('nsfwoid/score')
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
session = ClientSession()
class Score(web.View):
@time(REQUEST_TIME)
async def post(self):
request = self.request
data = await request.post()
try:
url = data["url"]
logger.info("Processing score request for URL: {0}".format(url))
image = await fetch(session, url)
nsfw_prob = await score(image)
return web.Response(text=nsfw_prob.astype(str))
except KeyError:
raise HTTPBadRequest(text="Missing `url` POST parameter")
except OSError as e:
if "cannot identify" in str(e):
class Mushaf(commands.Cog):
def __init__(self, bot):
self.bot = bot
self.session = ClientSession(loop=bot.loop)
async def _mushaf(self, ctx, ref, show_tajweed):
reference = QuranReference(ref)
async with self.session.get(
f'https://api.alquran.cloud/ayah/{reference.surah}:{reference.ayat_list}'
) as resp:
if resp.status != 200:
return await ctx.send(INVALID_VERSE)
data = await resp.json()
page = data['data']['page']
formatted_page = str(page).zfill(3)
if show_tajweed:
url = f'https://www.searchtruth.org/quran/images1/{formatted_page}.jpg'
else:
url = f'https://www.searchtruth.org/quran/images2/large/page-{formatted_page}.jpeg'
arabic_page_number = convert_to_arabic_number(str(page))
em = discord.Embed(title=f'Page {page}'
f'\n الصفحة{arabic_page_number}',
colour=0x006400)
em.set_author(name=f'Mushaf / مصحف', icon_url=ICON)
em.set_image(url=url)
await ctx.send(embed=em)
@commands.command(name="mushaf")
async def mushaf(self, ctx, ref: str, tajweed: bool = False):
await self._mushaf(ctx, ref, tajweed)
@mushaf.error
async def on_mushaf_error(self, ctx, error):
if isinstance(error, MissingRequiredArgument):
await ctx.send(INVALID_INPUT.format(ctx.prefix))
@cog_ext.cog_slash(
name="mushaf",
description="View an ayah on the mushaf.",
options=[
create_option(
name="reference",
description="The verse show on the mushaf, e.g. 1:4 or 2:255.",
option_type=3,
required=True),
create_option(
name="show_tajweed",
description=
"Should the mushaf highlight where tajweed rules apply?",
option_type=5,
required=False)
])
async def slash_mushaf(self,
ctx: SlashContext,
reference: str,
show_tajweed: bool = False):
await ctx.respond()
await self._mushaf(ctx, reference, show_tajweed)
class upnp:
def __init__(self, host, session: Optional[ClientSession] = None):
self.host = host
self.mute = False
self.volume = 0
if session:
self._session = session
self._managed_session = False
else:
self._session = ClientSession()
self._managed_session = True
def __enter__(self):
return self
async def _SOAPrequest(self, action, arguments, protocole):
headers = {
"SOAPAction":
'"urn:schemas-upnp-org:service:{protocole}:1#{action}"'.format(
action=action, protocole=protocole),
"content-type":
"text/xml",
}
body = """<?xml version="1.0" encoding="utf-8"?>
<s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
<s:Body>
<u:{action} xmlns:u="urn:schemas-upnp-org:service:{protocole}:1">
<InstanceID>0</InstanceID>
{arguments}
</u:{action}>
</s:Body>
</s:Envelope>""".format(action=action,
arguments=arguments,
protocole=protocole)
response = None
try:
with timeout(DEFAULT_TIMEOUT):
async with self._session.post(
f"http://{self.host}:9197/upnp/control/{protocole}1",
headers=headers,
data=body,
raise_for_status=True,
) as resp:
response = await resp.content.read()
except:
pass
return response
async def async_get_volume(self):
response = await self._SOAPrequest("GetVolume",
"<Channel>Master</Channel>",
"RenderingControl")
if response is not None:
volume_xml = response.decode("utf8")
tree = ET.fromstring(volume_xml)
for elem in tree.iter(tag="CurrentVolume"):
self.volume = elem.text
return self.volume
async def async_set_volume(self, volume):
await self._SOAPrequest(
"SetVolume",
"<Channel>Master</Channel><DesiredVolume>{}</DesiredVolume>".
format(volume),
"RenderingControl",
)
async def async_get_mute(self):
response = await self._SOAPrequest("GetMute",
"<Channel>Master</Channel>",
"RenderingControl")
if response is not None:
# mute_xml = response.decode('utf8')
tree = ET.fromstring(response.decode("utf8"))
mute = 0
for elem in tree.iter(tag="CurrentMute"):
mute = elem.text
if int(mute) == 0:
self.mute = False
else:
self.mute = True
return self.mute
async def async_set_current_media(self, url):
""" Set media to playback and play it."""
try:
await self._SOAPrequest(
"SetAVTransportURI",
"<CurrentURI>{url}</CurrentURI><CurrentURIMetaData></CurrentURIMetaData>"
.format(url=url),
"AVTransport",
)
await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport")
except Exception:
pass
async def async_play(self):
""" Play media that was already set as current."""
try:
await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport")
except Exception:
pass
class Client(object):
def __init__(self, url, dumper=None, loop=None):
self.url = url
self.dumper = dumper
if not loop:
loop = asyncio.get_event_loop()
if not self.dumper:
self.dumper = json.dumps
self.client = ClientSession(
loop=loop,
headers={'content-type': 'application/json'})
def __del__(self):
self.client.close()
def __encode(self, method, params=None, id=None):
try:
data = self.dumper({
"jsonrpc": "2.0",
"id": id,
"method": method,
"params": params
})
except Exception as e:
raise Exception("Can not encode: {}".format(e))
return data
@asyncio.coroutine
def call(self, method, params=None, id=None, schem=None):
if not id:
id = uuid4().hex
try:
resp = yield from self.client.post(
self.url, data=self.__encode(method, params, id))
except Exception as err:
raise Exception(err)
if 200 != resp.status:
raise InvalidResponse(
"Error, server retunrned: {status}".format(status=resp.status))
try:
data = yield from resp.json()
except Exception as err:
raise InvalidResponse(err)
try:
validate(data, ERR_JSONRPC20)
return Response(**data)
except ValidationError:
# Passing data to validate response.
# Good if does not valid to ERR_JSONRPC20 object.
pass
except Exception as err:
raise InvalidResponse(err)
try:
validate(data, RSP_JSONRPC20)
if id != data['id']:
raise InvalidResponse(
"Rsponse id {local} not equal {remote}".format(
local=id, remote=data['id']))
except Exception as err:
raise InvalidResponse(err)
if schem:
try:
validate(data['result'], schem)
except ValidationError as err:
raise InvalidResponse(err)
except Exception as err:
raise InternalError(err)
return Response(**data)
async def main():
connector = TCPConnector(limit=None)
async with ClientSession(connector=connector) as session:
await run(session, int(sys.argv[1]))
async def draw_profile(user: Member, session: ClientSession) -> Image:
name_fnt = ImageFont.truetype(font_heavy_file, 30)
name_u_fnt = ImageFont.truetype(font_unicode_file, 30)
title_fnt = ImageFont.truetype(font_heavy_file, 22)
title_u_fnt = ImageFont.truetype(font_unicode_file, 23)
label_fnt = ImageFont.truetype(font_bold_file, 18)
exp_fnt = ImageFont.truetype(font_bold_file, 13)
large_fnt = ImageFont.truetype(font_thin_file, 33)
rep_fnt = ImageFont.truetype(font_heavy_file, 26)
rep_u_fnt = ImageFont.truetype(font_unicode_file, 30)
text_fnt = ImageFont.truetype(font_file, 14)
text_u_fnt = ImageFont.truetype(font_unicode_file, 14)
symbol_u_fnt = ImageFont.truetype(font_unicode_file, 15)
def _write_unicode(text, init_x, y, font, unicode_font, fill):
write_pos = init_x
for char in text:
if char.isalnum() or char in string.punctuation or char in string.whitespace:
draw.text((write_pos, y), char, font=font, fill=fill)
write_pos += font.getsize(char)[0]
else:
draw.text((write_pos, y), u"{}".format(char), font=unicode_font, fill=fill)
write_pos += unicode_font.getsize(char)[0]
user_info = db.user(user)
# get urls
bg_url = await user_info.profile_background()
profile_url = user.avatar_url
async with session.get(bg_url) as r:
bg_image = Image.open(BytesIO(await r.content.read())).convert("RGBA")
async with session.get(profile_url) as r:
profile_image = Image.open(BytesIO(await r.content.read())).convert("RGBA")
# COLORS
white_color = (240, 240, 240, 255)
rep_fill = await user_info.rep_color()
badge_fill = await user_info.badge_col_color()
info_fill = await user_info.profile_info_color()
info_fill_tx = (*info_fill[:3], 150)
exp_fill = await user_info.profile_exp_color()
title = await user_info.title()
info = await user_info.info()
total_exp = await user_info.total_exp()
badges = await user_info.badges()
rep = await user_info.rep()
if badge_fill == (128, 151, 165, 230):
level_fill = white_color
else:
level_fill = _contrast(exp_fill, rep_fill, badge_fill)
# set canvas
bg_color = (255, 255, 255, 0)
result = Image.new("RGBA", (340, 390), bg_color)
process = Image.new("RGBA", (340, 390), bg_color)
# draw
draw = ImageDraw.Draw(process)
# puts in background
bg_image = bg_image.resize((340, 340), Image.ANTIALIAS)
bg_image = bg_image.crop((0, 0, 340, 305))
result.paste(bg_image, (0, 0))
# draw filter
draw.rectangle([(0, 0), (340, 340)], fill=(0, 0, 0, 10))
draw.rectangle([(0, 134), (340, 325)], fill=info_fill_tx) # general content
# draw profile circle
multiplier = 8
lvl_circle_dia = 116
circle_left = 14
circle_top = 48
raw_length = lvl_circle_dia * multiplier
# create mask
mask = Image.new("L", (raw_length, raw_length), 0)
draw_thumb = ImageDraw.Draw(mask)
draw_thumb.ellipse((0, 0) + (raw_length, raw_length), fill=255, outline=0)
# border
lvl_circle = Image.new("RGBA", (raw_length, raw_length))
draw_lvl_circle = ImageDraw.Draw(lvl_circle)
draw_lvl_circle.ellipse(
[0, 0, raw_length, raw_length], fill=(255, 255, 255, 255), outline=(255, 255, 255, 250)
)
# put border
lvl_circle = lvl_circle.resize((lvl_circle_dia, lvl_circle_dia), Image.ANTIALIAS)
lvl_bar_mask = mask.resize((lvl_circle_dia, lvl_circle_dia), Image.ANTIALIAS)
process.paste(lvl_circle, (circle_left, circle_top), lvl_bar_mask)
# put in profile picture
total_gap = 6
border = int(total_gap / 2)
profile_size = lvl_circle_dia - total_gap
mask = mask.resize((profile_size, profile_size), Image.ANTIALIAS)
profile_image = profile_image.resize((profile_size, profile_size), Image.ANTIALIAS)
process.paste(profile_image, (circle_left + border, circle_top + border), mask)
# write label text
white_color = (240, 240, 240, 255)
light_color = (160, 160, 160, 255)
dark_color = (35, 35, 35, 255)
head_align = 140
# determine info text color
info_text_color = _contrast(info_fill, white_color, dark_color)
_write_unicode(
_truncate_text(user.name, 22).upper(),
head_align,
142,
name_fnt,
name_u_fnt,
info_text_color,
) # NAME
_write_unicode(
title.upper(), head_align, 170, title_fnt, title_u_fnt, info_text_color
)
# draw divider
draw.rectangle([(0, 323), (340, 324)], fill=(0, 0, 0, 255)) # box
# draw text box
draw.rectangle(
[(0, 324), (340, 390)], fill=(info_fill[0], info_fill[1], info_fill[2], 255)
) # box
_write_unicode("❤", 257, 9, rep_fnt, rep_u_fnt, info_text_color)
draw.text(
(_center(278, 340, str(rep), rep_fnt), 10),
str(rep),
font=rep_fnt,
fill=info_text_color,
) # Exp Text
label_align = 362 # vertical
draw.text(
(_center(0, 140, " RANK", label_fnt), label_align),
" RANK",
font=label_fnt,
fill=info_text_color,
) # Rank
draw.text(
(_center(0, 340, " LEVEL", label_fnt), label_align),
" LEVEL",
font=label_fnt,
fill=info_text_color,
) # Exp
draw.text(
(_center(200, 340, "BALANCE", label_fnt), label_align),
"BALANCE",
font=label_fnt,
fill=info_text_color,
) # Credits
if "linux" in platform.system().lower():
global_symbol = u"\U0001F30E "
_write_unicode(
global_symbol, 36, label_align + 5, label_fnt, symbol_u_fnt, info_text_color
) # Symbol
_write_unicode(
global_symbol, 134, label_align + 5, label_fnt, symbol_u_fnt, info_text_color
) # Symbol
# userinfo
global_rank = f"#{await _find_global_rank(user)}"
global_level = str(_find_level(total_exp))
draw.text(
(_center(0, 140, global_rank, large_fnt), label_align - 27),
global_rank,
font=large_fnt,
fill=info_text_color,
) # Rank
draw.text(
(_center(0, 340, global_level, large_fnt), label_align - 27),
global_level,
font=large_fnt,
fill=info_text_color,
) # Exp
# draw level bar
exp_font_color = _contrast(exp_fill, light_color, dark_color)
exp_frac = int(total_exp - _level_exp(int(global_level)))
exp_total = _required_exp(int(global_level) + 1)
bar_length = int(exp_frac / exp_total * 340)
draw.rectangle(
[(0, 305), (340, 323)], fill=(level_fill[0], level_fill[1], level_fill[2], 245)
) # level box
draw.rectangle(
[(0, 305), (bar_length, 323)], fill=(exp_fill[0], exp_fill[1], exp_fill[2], 255)
) # box
exp_text = "{}/{}".format(exp_frac, exp_total) # Exp
draw.text(
(_center(0, 340, exp_text, exp_fnt), 305),
exp_text,
font=exp_fnt,
fill=exp_font_color,
) # Exp Text
credit_txt = f"${await bank.get_balance(user)}"
draw.text(
(_center(200, 340, credit_txt, large_fnt), label_align - 27),
_truncate_text(credit_txt, 18),
font=large_fnt,
fill=info_text_color,
) # Credits
if title == "":
offset = 170
else:
offset = 195
margin = 140
txt_color = _contrast(info_fill, white_color, dark_color)
for line in textwrap.wrap(info, width=32):
_write_unicode(line, margin, offset, text_fnt, text_u_fnt, txt_color)
offset += text_fnt.getsize(line)[1] + 2
# sort badges
priority_badges = []
for badgename in badges.keys():
badge = badges[badgename]
priority_num = badge["priority_num"]
if priority_num != 0 and priority_num != -1:
priority_badges.append((badge, priority_num))
sorted_badges = sorted(priority_badges, key=operator.itemgetter(1), reverse=True)
# TODO: simplify this. it shouldn't be this complicated... sacrifices conciseness for customizability
if (await db.badge_type()) == "circles":
# circles require antialiasing
vert_pos = 172
right_shift = 0
left = 9 + right_shift
size = 38
total_gap = 4 # /2
hor_gap = 6
vert_gap = 6
border_width = int(total_gap / 2)
multiplier = 6 # for antialiasing
raw_length = size * multiplier
mult = [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1), (0, 2), (1, 2), (2, 2)]
for num in range(9):
coord = (
left + int(mult[num][0]) * int(hor_gap + size),
vert_pos + int(mult[num][1]) * int(vert_gap + size),
)
if num < len(sorted_badges[:9]):
pair = sorted_badges[num]
badge = pair[0]
bg_color = badge["bg_img"]
border_color = badge["border_color"]
# draw mask circle
mask = Image.new("L", (raw_length, raw_length), 0)
draw_thumb = ImageDraw.Draw(mask)
draw_thumb.ellipse((0, 0) + (raw_length, raw_length), fill=255, outline=0)
# determine image or color for badge bg
if await self._valid_image_url(bg_color):
# get image
async with session.get(bg_color) as r:
badge_image = Image.open(BytesIO(await r.content.read())).convert("RGBA")
badge_image = badge_image.resize((raw_length, raw_length), Image.ANTIALIAS)
# structured like this because if border = 0, still leaves outline.
if border_color:
square = Image.new("RGBA", (raw_length, raw_length), border_color)
# put border on ellipse/circle
output = ImageOps.fit(
square, (raw_length, raw_length), centering=(0.5, 0.5)
)
output = output.resize((size, size), Image.ANTIALIAS)
outer_mask = mask.resize((size, size), Image.ANTIALIAS)
process.paste(output, coord, outer_mask)
# put on ellipse/circle
output = ImageOps.fit(
badge_image, (raw_length, raw_length), centering=(0.5, 0.5)
)
output = output.resize(
(size - total_gap, size - total_gap), Image.ANTIALIAS
)
inner_mask = mask.resize(
(size - total_gap, size - total_gap), Image.ANTIALIAS
)
process.paste(
output,
(coord[0] + border_width, coord[1] + border_width),
inner_mask,
)
else:
# put on ellipse/circle
output = ImageOps.fit(
badge_image, (raw_length, raw_length), centering=(0.5, 0.5)
)
output = output.resize((size, size), Image.ANTIALIAS)
outer_mask = mask.resize((size, size), Image.ANTIALIAS)
process.paste(output, coord, outer_mask)
else:
plus_fill = exp_fill
# put on ellipse/circle
plus_square = Image.new("RGBA", (raw_length, raw_length))
plus_draw = ImageDraw.Draw(plus_square)
plus_draw.rectangle(
[(0, 0), (raw_length, raw_length)],
fill=(info_fill[0], info_fill[1], info_fill[2], 245),
)
# draw plus signs
margin = 60
thickness = 40
v_left = int(raw_length / 2 - thickness / 2)
v_right = v_left + thickness
v_top = margin
v_bottom = raw_length - margin
plus_draw.rectangle(
[(v_left, v_top), (v_right, v_bottom)],
fill=(plus_fill[0], plus_fill[1], plus_fill[2], 245),
)
h_left = margin
h_right = raw_length - margin
h_top = int(raw_length / 2 - thickness / 2)
h_bottom = h_top + thickness
plus_draw.rectangle(
[(h_left, h_top), (h_right, h_bottom)],
fill=(plus_fill[0], plus_fill[1], plus_fill[2], 245),
)
# put border on ellipse/circle
output = ImageOps.fit(
plus_square, (raw_length, raw_length), centering=(0.5, 0.5)
)
output = output.resize((size, size), Image.ANTIALIAS)
outer_mask = mask.resize((size, size), Image.ANTIALIAS)
process.paste(output, coord, outer_mask)
result = Image.alpha_composite(result, process)
result = _add_corners(result, 25)
return result
async def generate_requests() -> list:
urls = mapUrls.keys()
async with ClientSession(connector=TCPConnector(
verify_ssl=False)) as session:
responses = await fetch_all(session, urls)
return responses
async def run(args: argparse):
"""
Metodo que realizada una peticion asincrona post por cada elemento de los diccionarios proporcionados
:param args:
:return:
"""
name_username: str
name_password: str
my_json: Dict[str, str] = json.loads(args.json)
keys_json = list(my_json)
# si el json tiene 1 valor se entiende que es la pass, si tiene 2 hay user y pass
if len(keys_json) >= 2:
name_username = keys_json[0]
name_password = keys_json[1]
elif len(keys_json) == 1:
name_username = ''
name_password = keys_json[0]
else:
logger.critical('json is not valid')
sys.exit(1)
dict_users: List = list()
dict_passwords: List = list()
# creamos un diccionaro de users
if args.Login is None:
dict_users.append(args.login)
else:
file_username: Path = Path(args.Login)
if file_username.exists():
dict_users = file_username.read_text().split('\n')
dict_users.remove('') # eliminar lineas en blanco
else:
logger.critical(f'File "{args.Login}" not exits')
sys.exit(1)
# creamos un diccionaro de passwords
if args.Password is None:
dict_passwords.append(args.password)
else:
file_password: Path = Path(args.Password)
if file_password.exists():
dict_passwords = file_password.read_text().split('\n')
else:
logger.critical(f'File "{args.Password}" not exits')
sys.exit(1)
is_online(args.target)
len_user = len(dict_users)
len_pwd = len(dict_passwords)
user: str
pwd: str
cont: int = 1
tasks: List = list()
# create instance of Semaphore
sem: asyncio.locks.Semaphore = asyncio.Semaphore(1000)
# Create client session that will ensure we dont open new connection per each request.
async with ClientSession() as session:
for user in dict_users:
# if len(user) == 0: # si existe alguna liena en blanco la omitimos, ya se hace al crear la lista
# continue
# Actualizamos el username si es necesario ponerlo, ya se hace al crear la lista
if name_username in my_json.keys():
my_json[name_username] = user
for pwd in dict_passwords:
# Actualizamos la contraseña
if name_password in my_json.keys():
my_json[name_password] = pwd
# pass Semaphore and session to every GET request
update_json = json.dumps(
my_json
) # si envio el json como dict llega vacio a la funcion (i dont known)
task = asyncio.ensure_future(
bound_fetch(sem, session, args, user, pwd, update_json,
(len_user * len_pwd), cont))
tasks.append(task)
cont += 1
responses = asyncio.gather(*tasks)
await responses
def __init__(self, bot):
self.bot = bot
self.session = ClientSession(loop=bot.loop)
async def __create_aiohttp_client_session(self):
if not self._aiohttp_client_session:
ck = CookieJar(unsafe=True)
self._aiohttp_client_session = ClientSession(cookie_jar=ck)
class Database():
ins = None
init = False
def __new__(cls, endpoint:str=None) -> Any:
if not cls.ins:
cls.ins = super().__new__(cls)
return cls.ins
def __init__(self, endpoint:str=None) -> None:
if Database.init:
return
Database.init = True
self.sesison = ClientSession() # TOOD: do timeout
self.endpoint = endpoint
log.debug(f"endpoint set to {self.endpoint}")
async def perform(self, method:str, endpoint:str, *, params:dict=None, json:dict=None) -> Optional[dict]:
try:
async with self.sesison.request(method, f"{self.endpoint}{endpoint}", params=params, json=json) as resp:
payload = await resp.json(content_type=None)
log.debug(f"{method} {endpoint} (params: {params}, json: {json}) -> {resp.status}")
if resp.status == 200:
return payload
else:
raise DatabaseError()
except DatabaseError as e:
raise e
except Exception as e:
log.error(f"Further information: ", exc_info=True)
raise e
get = partialmethod(perform, "GET")
post = partialmethod(perform, "POST")
delete = partialmethod(perform, "DELETE")
async def get_playlists(self) -> Optional[dict]:
return await self.get("/playlist")
async def get_playlist(self, name:str, *, limit:int=1000) -> Optional[dict]:
return await self.get(f"/playlist/{name}", params={"limit": limit})
async def create_playlist(self, name:str) -> None:
await self.post("/playlist", params={"name": name})
async def delete_playlist(self, name:str) -> None:
await self.delete(f"/playlist", params={"name": name})
async def add_to_playlist(self, name:str, uris:Sequence[str]) -> None:
await self.post(f"/playlist/{name}", json={"entries": uris})
async def delete_from_playlist(self, name:str, uri:str) -> None:
await self.delete(f"/playlist/{name}", params={"uri": uri})
async def get_likes(self, limit:int=1000) -> Optional[dict]:
return await self.get("/likes", params={"limit": limit})
async def toggle_like(self, uri:str, like:bool) -> None:
await self.post("/likes", params={"uri": uri, "like": 1 if like else 0})
async def is_liked(self, uri:str) -> Optional[dict]:
return await self.get("/likes/resolve", params={"uri": uri})
async def get_cache(self, uri:str) -> Optional[dict]:
return await self.get("/cache", params={"uri": uri})
async def store_cache(self, entries:Sequence[dict]) -> None:
await self.post("/cache", json={"entries": entries})
async def update_gpm(self, entries:Sequence[dict], user:str) -> None:
await self.post("/gpm/update", params={"name": user}, json={"entries": entries})
async def search_gpm(self, query:str, *, limit=100) -> Optional[dict]:
return await self.get("/gpm/search", params={"limit": limit, "query": query})
async def resolve_gpm(self, uri:str) -> Optional[dict]:
return await self.get("/gpm", params={"uri": uri})
async def _get_async_session(endpoint_uri: URI) -> ClientSession:
cache_key = generate_cache_key(endpoint_uri)
if cache_key not in _async_session_cache:
await cache_async_session(endpoint_uri, ClientSession(raise_for_status=True))
return _async_session_cache.get_cache_entry(cache_key)
async def _get(url: str, session: aiohttp.ClientSession):
async with session.get(url, headers=_HEADERS) as response:
assert response.status == 200
return await response.json(loads=ujson.loads)
async def fetch_playlist(playlistId: str, chunk_size: int = 1024):
"""
Fetch all metadata of a YouTube playlist.
Performs multiple asynchronous HTTP requests to obtain all metadata and
playlist items. Does not provide rich YouTube metadata: fields that are
included in browse_request queries for modern JS YT-Clients.
Result:
{
'id': string,
'title': string,
'description': string,
'thumbnail': string,
'length': integer,
'views': integer,
'uploader': {
'name': string,
'url': string
},
'items': [
{
'id': string,
'title': string,
'uploader': {
'name': string,
'url': string
},
'lengthSeconds': integer,
}
]
}
"""
#
# Initialize the result
#
playlist = {'id': playlistId, 'items': []}
#
# Build the headers for the ClientSession.
#
headers = get_default_headers()
# Only accept HTML.
headers['Accept'] = 'text/html'
#
# Retrieve landing page.
#
# Open a auto-raising ClientSession with default cookie handling.
async with ClientSession(headers=headers,
raise_for_status=True) as session:
# Step 1: Get the initial landing page.
async with session.get('https://www.youtube.com/playlist',
params={'list': playlistId}) as response:
# Assert that this really worked the way we wanted.
assert response.status == 200
assert response.content_type == 'text/html'
encoding = response.get_encoding()
assert is_valid_encoding(encoding)
# Retrieve the '//div[@id=""]' node.
is_content = lambda x: (x.tag, x.get('id')) == ('div', '')
parser = etree.HTMLPullParser(events=('start', 'end'))
content, discard = None, True
while content is None and not response.content.at_eof():
# Feed the parser the next chunk of data.
parser.feed(
(await response.content.read(chunk_size)).decode(encoding))
for event, node in parser.read_events():
if event == 'start':
if is_content(node):
# Content node reached, stop discarding.
discard = False
continue
if is_content(node):
# Content node finished, exit.
content = node
break
if discard:
# Discard everything before this point.
node.clear()
for ancestor in node.xpath('ancestor-or-self::*'):
while ancestor.getprevious() is not None:
del ancestor.getparent()[0]
#
# Parse the playlist header.
#
pl_header = content[0]
assert pl_header.get('id') == 'pl-header'
# Get the thumbnail.
pl_thumb = pl_header[0][0]
assert pl_thumb.tag == 'img'
playlist['thumbnail'] = pl_thumb.get('src')
# Get the title.
pl_title = pl_header[1][0]
assert pl_title.tag == 'h1'
playlist['title'] = pl_title.text.strip()
# Get the uploader.
pl_uploader = pl_header[1][1][0][0]
assert pl_uploader.tag == 'a'
playlist['uploader'] = {
'name': pl_uploader.text,
'url': 'https://www.youtube.com' + pl_uploader.get('href')
}
# Get the length.
pl_length = pl_header[1][1][1]
assert pl_length.tag == 'li'
playlist['length'] = parse_int(pl_length.text, aggressive=True)
# Get the view count.
pl_views = pl_header[1][1][2]
assert pl_views.tag == 'li'
playlist['views'] = parse_int(pl_views.text, aggressive=True)
# Get the description.
pl_description = pl_header[1][2][0]
assert pl_description.tag == 'span'
playlist['description'] = pl_description.text.strip()
#
# Parse the playlist items.
#
def parse_item(node):
item = {'id': node.get('data-video-id')}
# Get the video thumbnail.
vid_thumb = node[2][0][0][0][0][0][0]
assert vid_thumb.tag == 'img'
item['thumbnail'] = vid_thumb.get('data-thumb')
# Get video title.
vid_title = node[3][0]
assert vid_title.tag == 'a'
item['title'] = vid_title.text.strip()
# Get video uploader.
vid_uploader = node[3][1][0]
assert vid_uploader.tag == 'a'
item['uploader'] = {
'name': vid_uploader.text,
'url': 'https://www.youtube.com' + vid_uploader.get('href')
}
# Get video length.
vid_length = node[6][0][0][0]
assert vid_length.tag == 'span'
item['lengthSeconds'] = parse_ts(vid_length.text)
return item
pl_items = content[1][0][0][0][0]
assert pl_items.get('id') == 'pl-load-more-destination'
playlist['items'] += map(parse_item, pl_items)
#
# Fetch and parse all continuations.
#
load_more = index_s(content, 1, 0, 0, 1)
assert load_more.tag == 'button'
load_more = load_more.get(
'data-uix-load-more-href') if load_more is not None else None
while load_more is not None:
# Request the continuation contents.
async with session.get('https://www.youtube.com' + load_more,
headers={'Accept':
'application/json'}) as response:
# Assert that this really worked the way we wanted.
assert response.status == 200
assert response.content_type == 'application/json'
encoding = response.get_encoding()
assert is_valid_encoding(encoding)
# Parse the result data (large!)
# TODO: This ought to be streamed as well.
data = await response.json()
assert 'content_html' in data
assert 'load_more_widget_html' in data
# Parse all new items.
parser = etree.HTMLPullParser(events=('end', ))
parser.feed(data['content_html'])
for _, node in parser.read_events():
if node.tag == 'tr':
playlist['items'] += [parse_item(node)]
# Discard everything before this point.
node.clear()
for ancestor in node.xpath('ancestor-or-self::*'):
while ancestor.getprevious() is not None:
del ancestor.getparent()[0]
# Extract the next continuation link
match = re.search(r'data-uix-load-more-href=\"(.+?)\"',
data['load_more_widget_html'])
if match:
# Next continuation link.
load_more = match.group(1)
else:
# No more continuations.
load_more = None
return playlist
def __init__(self, username, password, timeout=10):
self._loop = asyncio.get_event_loop()
self._session = ClientSession(loop=self._loop, auth=BasicAuth(username, password))
self._timeout = timeout
async def get_json_content(url):
async with ClientSession() as session:
async with session.get(url) as resp:
print("Response:" + str(resp.status))
json_string = await resp.text()
return loads(json_string)
async def _auth_registry_request(url: URL, method: str, auth_headers: Dict,
session: ClientSession) -> Tuple[Dict, Dict]:
if not config.REGISTRY_AUTH or not config.REGISTRY_USER or not config.REGISTRY_PW:
raise exceptions.RegistryConnectionError(
"Wrong configuration: Authentication to registry is needed!")
# auth issue let's try some authentication get the auth type
auth_type = None
auth_details: Dict[str, str] = {}
for key in auth_headers:
if str(key).lower() == "www-authenticate":
auth_type, auth_value = str(auth_headers[key]).split(" ", 1)
auth_details = {
x.split("=")[0]: x.split("=")[1].strip('"')
for x in auth_value.split(",")
}
break
if not auth_type:
raise exceptions.RegistryConnectionError(
"Unknown registry type: cannot deduce authentication method!")
auth = BasicAuth(login=config.REGISTRY_USER, password=config.REGISTRY_PW)
# bearer type, it needs a token with all communications
if auth_type == "Bearer":
# get the token
token_url = URL(auth_details["realm"]).with_query(
service=auth_details["service"], scope=auth_details["scope"])
async with session.get(token_url, auth=auth) as token_resp:
if not token_resp.status == HTTPStatus.OK:
raise exceptions.RegistryConnectionError(
"Unknown error while authentifying with registry: {}".
format(str(token_resp)))
bearer_code = (await token_resp.json())["token"]
headers = {"Authorization": "Bearer {}".format(bearer_code)}
async with getattr(session,
method.lower())(url,
headers=headers) as resp_wtoken:
if resp_wtoken.status == HTTPStatus.NOT_FOUND:
logger.exception("path to registry not found: %s", url)
raise exceptions.ServiceNotAvailableError(str(url))
if resp_wtoken.status > 399:
logger.exception(
"Unknown error while accessing with token authorized registry: %s",
str(resp_wtoken),
)
raise exceptions.RegistryConnectionError(str(resp_wtoken))
resp_data = await resp_wtoken.json(content_type=None)
resp_headers = resp_wtoken.headers
return (resp_data, resp_headers)
elif auth_type == "Basic":
# basic authentication should not be since we tried already...
async with getattr(session, method.lower())(url,
auth=auth) as resp_wbasic:
if resp_wbasic.status == HTTPStatus.NOT_FOUND:
logger.exception("path to registry not found: %s", url)
raise exceptions.ServiceNotAvailableError(str(url))
if resp_wbasic.status > 399:
logger.exception(
"Unknown error while accessing with token authorized registry: %s",
str(resp_wbasic),
)
raise exceptions.RegistryConnectionError(str(resp_wbasic))
resp_data = await resp_wbasic.json(content_type=None)
resp_headers = resp_wbasic.headers
return (resp_data, resp_headers)
raise exceptions.RegistryConnectionError(
f"Unknown registry authentification type: {url}")
class SpectateMotor:
def __init__(self, bot):
self.bot = bot
self.session = ClientSession(loop=self.bot.loop)
self.map = None
self.bots = []
self.turns = 0
self.latest = {}
self.update_task = None
async def interrupt_spectating(self):
await self.map.delete()
self.update_task.cancel()
self.update_task = None
def update_turns(self):
self.turns += 1
def interrupt_variables(self):
self.bots = []
self.turns = 0
self.latest = {}
async def do_restart(self, ctx, reason):
counter = await ctx.send(
f':warning: | **Initialized restart due to {reason}**')
try:
await self.interrupt_spectating()
self.interrupt_variables()
except:
pass
async for m in self.bot.get_channel(505815417269518346).history(
limit=1000):
try:
if m.id != counter.id:
await m.delete()
except:
continue
await asyncio.sleep(1)
for count in range(5):
await counter.edit(
content=f':warning: | **Restarting in {abs(count - 5)}s**')
await asyncio.sleep(1)
await counter.delete()
os.execl(sys.executable, sys.executable, *sys.argv)
async def updater(self, channel):
await self.bot.wait_until_ready()
self.map = await self.bot.get_channel(channel).send('Loading...')
while not self.bot.is_closed():
await asyncio.sleep(3)
print(f'Getting map...\n----------')
async with self.session.get(
"https://league.travitia.xyz/map") as r:
map = await r.json()
if map == self.latest:
continue
self.latest = map
out = ""
try:
for row in map:
out = f"{out}\n{''.join(['â–ˆ' if not i else i['name'][0] for i in row])}"
print('Getting bots alive...\n----------')
async with self.session.get(
"https://league.travitia.xyz/bots") as bots:
self.bots = self.bots = sorted((await bots.json()),
key=lambda d: -d['hp'])
except:
print(
f'Game end. Ended in {self.turns} Generations...\n----------'
)
await self.map.edit(
content=
f"```fix\n---GAME ENDED---\n{map['text'].capitalize()}\nTotal Generations: {self.turns}\nThis Game Took: ~{datetime.timedelta(seconds=self.turns * 3)}```"
)
return self.interrupt_variables()
self.update_turns()
hp = '\n'.join(
[f"{bot['name']}: {bot['hp']} HP" for bot in self.bots])
await self.map.edit(
content=f'```fix\n{out}\n\n{hp}\n\nGeneration {self.turns}```')
@commands.command(aliases=['s', 'st', 'sp', 'spect'])
async def spectate(self, ctx):
"""Spectate The Game in LIVE Mode"""
_ = await ctx.send('Trying to connect... Check <#505815417269518346>')
if self.update_task is None:
self.update_task = self.bot.loop.create_task(
self.updater(505815417269518346))
await _.edit(
content="🔴 **NOW LIVE!** Check <#505815417269518346>")
else:
await _.edit(
content='Already spectating... Check <#505815417269518346>')
@commands.command(aliases=['stahp'])
async def stop(self, ctx):
"""Stop Spectating The Game"""
if self.update_task is None:
await ctx.send('Not spectating...')
else:
try:
await self.interrupt_spectating()
self.interrupt_variables()
except:
pass
await ctx.send('Stopped spectating...')
@commands.command(aliases=['newgame', 'ng'])
async def new(self, ctx):
"""Starts a New Game"""
async with self.session.get("https://league.travitia.xyz/new") as req:
if (await req.json())["status"] == "error":
await ctx.send("Game already running... you can't start new..."
)
else:
await ctx.send("New game successfully created...")
def __init__(self, gcp_session: ClientSession(), token: [str], logging_context: LoggingContext):
self.gcp_session = gcp_session
self.token = token
self.logging_context = logging_context
async def get_json_data(url: str) -> dict:
async with ClientSession(connector=TCPConnector(ssl=False)) as session:
async with session.get(url=url) as response:
return await response.json()
