def go(dirname, filename): ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ssl_ctx.load_cert_chain( os.path.join(dirname, 'sample.crt'), os.path.join(dirname, 'sample.key') ) app, _, url = yield from self.create_server( 'GET', '/static/' + filename, ssl_ctx=ssl_ctx ) app.router.add_static('/static', dirname) conn = TCPConnector(verify_ssl=False, loop=self.loop) session = ClientSession(connector=conn) resp = yield from session.request('GET', url) self.assertEqual(200, resp.status) txt = yield from resp.text() self.assertEqual('file content', txt.rstrip()) ct = resp.headers['CONTENT-TYPE'] self.assertEqual('application/octet-stream', ct) self.assertEqual(resp.headers.get('CONTENT-ENCODING'), None) resp.close() resp = yield from session.request('GET', url + 'fake') self.assertEqual(404, resp.status) resp.close() resp = yield from session.request('GET', url + '/../../') self.assertEqual(404, resp.status) resp.close()
def github_action(action, plugin, config, sort_by='updated'): url = API_URL.format( api=API_URL, user=config['user'], repo=config['repository'], action=action, ) query = { 'sort': sort_by, 'direction': 'desc', 'sha': config.get('branch', 'master') } headers = {'Accept': 'application/vnd.github.v3+json'} etag = plugin.temp.get(action) if etag is not None: headers['If-None-Match'] = etag session = ClientSession() try: resp = yield from asyncio.wait_for( session.get(url, params=query, headers=headers), timeout=5 ) try: plugin.temp[action] = resp.headers.get('etag') if resp.status != 200 or etag is None: # etag must be cached first raise NothingChangeException(etag) data = yield from resp.json() finally: resp.close() finally: session.close() return data[0]
def go(): _, srv, url = yield from self.create_server(None, '/', None) client = ClientSession(loop=self.loop) resp = yield from client.get(url) self.assertEqual(404, resp.status) yield from resp.release() client.close()
def go(): _, srv, url = yield from self.create_server('GET', '/', handler) client = ClientSession(loop=self.loop) resp = yield from client.get(url) self.assertEqual(200, resp.status) data = yield from resp.read() self.assertEqual(b'xyz', data) yield from resp.release()
def go(): _, srv, url = yield from self.create_server('GET', '/', handler) client = ClientSession(loop=self.loop) resp = yield from client.get(url) self.assertEqual(200, resp.status) data = yield from resp.read() self.assertEqual(b'mydata', data) self.assertEqual(resp.headers.get('CONTENT-ENCODING'), 'deflate') yield from resp.release() client.close()
def get_html(url): session = ClientSession() try: resp = yield from session.get(url) try: raw_data = yield from resp.read() data = raw_data.decode('utf-8', errors='xmlcharrefreplace') return fromstring_to_html(data) finally: resp.close() finally: session.close()
def __init__(self, configuration): self._limit_sleep_time_coefficient = configuration \ .instagram_limit_sleep_time_coefficient self._limit_sleep_time_min = configuration \ .instagram_limit_sleep_time_min self._success_sleep_time_coefficient = configuration \ .instagram_success_sleep_time_coefficient self._success_sleep_time_max = configuration \ .instagram_success_sleep_time_max self._success_sleep_time_min = configuration \ .instagram_success_sleep_time_min self._limit_sleep_time = self._limit_sleep_time_min self._success_sleep_time = self._success_sleep_time_max self._username = configuration.instagram_username self._password = configuration.instagram_password self._referer = BASE_URL self._session = ClientSession( cookies={ 'ig_pr': '1', 'ig_vw': '1920', }, headers={ 'User-Agent': USER_AGENT, 'X-Instagram-AJAX': '1', 'X-Requested-With': 'XMLHttpRequest', }, ) loop = asyncio.get_event_loop() loop.run_until_complete(self._do_login())
async def query_imdb(*, imdb_id: Optional[int], year: int, session: ClientSession) -> Dict[str, Any]: params = dict(i=f'tt{imdb_id:0>{IMDB_ID_LENGTH}}', y=year, plot='full', tomatoes='true', r='json') while True: attempt_num = 0 async with session.get(IMDB_API_URL, params=params) as response: attempt_num += 1 if response.status == A_TIMEOUT_OCCURRED: logger.debug(f'Attempt #{attempt_num} failed: ' f'server "{IMDB_API_URL}" answered with ' f'status code {A_TIMEOUT_OCCURRED}. ' f'Waiting {RETRY_INTERVAL_IN_SECONDS} second(s) ' 'before next attempt.') await sleep(RETRY_INTERVAL_IN_SECONDS) continue try: response_json = await response.json() except JSONDecodeError: logger.exception('') return dict() return response_json
async def check(self, aio_client: aiohttp.ClientSession, usernames: AbstractSet[str]) -> ni_abc.Status: base_url = "https://bugs.python.org/user?@template=clacheck&github_names=" url = base_url + ','.join(usernames) self.server.log("Checking CLA status: " + url) async with aio_client.get(url) as response: if response.status >= 300: msg = f'unexpected response for {response.url!r}: {response.status}' raise client.HTTPException(msg) # Explicitly decode JSON as b.p.o doesn't set the content-type as # `application/json`. results = json.loads(await response.text()) self.server.log("Raw CLA status: " + str(results)) status_results = [results[k] for k in results.keys() if k in usernames] self.server.log("Filtered CLA status: " + str(status_results)) if len(status_results) != len(usernames): raise ValueError("# of usernames don't match # of results " "({} != {})".format(len(usernames), len(status_results))) elif any(x not in (True, False, None) for x in status_results): raise TypeError("unexpected value in " + str(status_results)) if all(status_results): return ni_abc.Status.signed elif any(value is None for value in status_results): return ni_abc.Status.username_not_found else: return ni_abc.Status.not_signed
async def fetch_with_sleep(session: aiohttp.ClientSession, url: str, sleep_time: int) -> Response: async with session.get(url) as response: print(f"request {url} with {sleep_time}s") await sleep(sleep_time) print(f"request {url} sleep done") return response
async def _fetch_url( url: str, session: aiohttp.ClientSession, timeout: float = 10.0) -> str: with async_timeout.timeout(timeout): async with session.get(url) as response: response.raise_for_status() return await response.text()
class Bot: '''Base class of your bot. You should implement the ``on_message`` function. Text may be parsed using ``parse`` ''' def __init__(self): self.conversations = {} self.history = [] self._client = ClientSession() def __enter__(self): return self def __exit__(self, ex_type, ex_value, ex_tb): self._client.close() def close(self): self._client.close() async def parse(self, text): async with self._client.post( LUIS_ENDPOINT, text.encode('utf-8'), ) as resp: data = await resp.json() # TODO: Extract relevant information from data return data @abstractmethod async def on_message(self, conversation, text): pass async def _handler(self, request): return web.Response( b"Not implemented", ) async def _history(self, request): return web.Response( '\n'.join('<p>{!r}</p>'.format(h) for h in self.history).encode('utf-8') )
async def afetch(session: ClientSession, url: str): """ Asynchronous fetch. Do a GET request, return text, properly shutdown :param session: ClientSession object for aiohttp connection :param url: The URL we want :return: """ async with session.get(url) as response: return await response.text()
class Downloader(Actor): async def startup(self): self.session = ClientSession(loop=self.loop) @concurrent async def download_content(self, url): async with self.session.get(url) as response: content = await response.read() print('{}: {:.80}...'.format(url, content.decode())) return len(content) async def shutdown(self): self.session.close() async def __aenter__(self): return self async def __aexit__(self, exc_type, exc, tb): await self.close()
async def process_partition( loop: asyncio.BaseEventLoop, results_queue: asyncio.Queue, server_address: URL, http: aiohttp.ClientSession, partition: PointsPartition, mission_template: Template, mission_loader: str, mission_name: str, width: int, scale: int, ) -> Awaitable[None]: LOG.debug( f"query range [{partition.start}:{partition.end}] on server " f"{server_address}" ) file_name = f"{mission_name}_{partition.start}_{partition.end}.mis" missions_url = server_address / "missions" mission_dir_url = missions_url / "heightmap" mission_url = mission_dir_url / file_name points = ( index_to_point(i, width, scale) for i in range(partition.start, partition.end + 1) ) mission = mission_template.render( loader=mission_loader, points=points, ) data = FormData() data.add_field( 'mission', mission.encode(), filename=file_name, content_type='plain/text', ) await http.post(mission_dir_url, data=data) await http.post(mission_url / "load") await http.post(missions_url / "current" / "begin") async with http.get(server_address / "radar" / "stationary-objects") as response: data = await response.json() data = [ pack(HEIGHT_PACK_FORMAT, int(point['pos']['z'])) for point in data ] data = b''.join(data) await http.post(missions_url / "current" / "unload") await http.delete(mission_url) await results_queue.put((partition, data))
def __init__(self, config): config = self.trafaret.check(config) self.base_url = config['stream'] self.signature = HmacSha1Signature() self.session = ClientSession() self.oauth_client = oauth.Client( client_key=config['api_key'], client_secret=config['api_secret'], resource_owner_key=config['access_token'], resource_owner_secret=config['access_secret'] )
def __init__(self, url, dumper=None, loop=None): self.url = url self.dumper = dumper if not loop: loop = asyncio.get_event_loop() if not self.dumper: self.dumper = json.dumps self.client = ClientSession( loop=loop, headers={'content-type': 'application/json'})
async def close_self_reaction_pr(number: int, session: aiohttp.ClientSession): # PATCH /repos/:owner/:repo/pulls/:number pr_url = github_url( f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/pulls/{number}") await session.patch(pr_url, headers=HEADERS, data=b'{"state": "closed"}') # POST /repos/:owner/:repo/issues/:number/comments comment_url = github_url( f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/issues/{number}/comments") async with session.post(comment_url, headers=HEADERS, data=json.dumps({"body": "\> Reacting to your own PR"})) as resp: content = await resp.text() print(f"RESPONSE: {resp.status}, CONTENT: {content}")
def __init__(self, app, protocol="http"): self._app = app self._loop = loop = app.loop self.port = unused_port() self._handler = handler = app.make_handler() self._server = loop.run_until_complete(loop.create_server( handler, '127.0.0.1', self.port )) self._session = ClientSession(loop=self._loop) self._root = "{}://127.0.0.1:{}".format( protocol, self.port ) self._closed = False
class StreamingClient: filter_endpoint = "/1.1/statuses/filter.json" sample_endpoint = "/1.1/statuses/sample.json" predicates = ["follow", "locations", "track"] keys = ['language', 'delimited', 'stall_warnings'] trafaret = t.Dict({ 'api_key': t.String, 'api_secret': t.String, 'access_token': t.String, 'access_secret': t.String, 'stream': t.URL, }).ignore_extra('*') def __init__(self, config): config = self.trafaret.check(config) self.base_url = config['stream'] self.signature = HmacSha1Signature() self.session = ClientSession() self.oauth_client = oauth.Client( client_key=config['api_key'], client_secret=config['api_secret'], resource_owner_key=config['access_token'], resource_owner_secret=config['access_secret'] ) async def request(self, method, endpoint, params=None, headers=None): url = urljoin(self.base_url, endpoint) headers = (headers or {}) if method == 'POST': headers['Content-type'] = 'application/x-www-form-urlencoded' uri, signed_headers, body = self.oauth_client.sign(url, method, params, headers) logger.debug("PREPARED: %s %s %s", uri, signed_headers, body) resp = await self.session.request(method, uri, params=body, headers=signed_headers) return resp async def stream(self, queue, predicate, value, **kwargs): params = dict(((predicate, value,),)) while True: resp = await self.request('POST', self.filter_endpoint, params) while True: data = await resp.content.readline() message = data.strip() if message != b'': message = json.loads(message.decode('utf-8')) logger.debug("Got something %s", message['id']) await queue.put(message) await asyncio.sleep(10) def __del__(self): asyncio.ensure_future(self.session.close())
async def bfetch(session: ClientSession, url: str): """ Asynchronous binary fetch. Do a GET request, return binary data, properly shutdown :param session: ClientSession object for aiohttp connection :param url: The URL we want :return: """ async with session.get(url) as response: if response.status >= 400: LOGGER.error("Got status %s for GET %s", response.status, url) response.raise_for_status() return await response.read()
class Github: _BASE_URL = 'https://api.github.com' def __init__(self, username, password, timeout=10): self._loop = asyncio.get_event_loop() self._session = ClientSession(loop=self._loop, auth=BasicAuth(username, password)) self._timeout = timeout def close(self): self._session.close() async def fetch(self, url, params): with Timeout(self._timeout): async with self._session.get('{}{}'.format(self._BASE_URL, url), params=params) as response: return await response.json() async def search_repositories(self, language, pushed, sort, order): q = 'language:{}'.format(language) if pushed: q = '{} pushed:>={}'.format(q, pushed) params = {'q': q, 'sort': sort, 'order': order} return await self.fetch('/search/repositories', params=params)
async def get_imdb_id(article_title: str, *, session: ClientSession ) -> Optional[int]: params = dict(action='expandtemplates', text='{{IMDb title}}', prop='wikitext', title=article_title, format='json') async with session.get(WIKIPEDIA_API_URL, params=params) as response: response_json = await response.json() templates = response_json['expandtemplates'] imdb_link = templates.get('wikitext', '') search_res = IMDB_ID_RE.search(imdb_link) if search_res is None: return None return int(search_res.group(0))
class RestClient(): def __init__(self): self.client = ClientSession() def close(self): self.client.close() @asyncio.coroutine def post(self, url, headers={}): if 'content-type' not in headers: headers['content-type'] = 'application/json' resp = yield from self.client.post(url, headers=headers) data = yield from resp.read() yield from resp.release() return data.decode('utf-8') @asyncio.coroutine def get(self, url, headers={}, to_json=False): resp = yield from self.client.post(url) data = yield from resp.read() yield from resp.release() if to_json: data = json.loads(data.decode('utf-8')) return data @asyncio.coroutine def put(self, url, headers={}): resp = yield from self.client.post(url) data = yield from resp.read() yield from resp.release() return json.loads(data) @asyncio.coroutine def delete(self, url): resp = yield from self.client.post(url) data = yield from resp.read() yield from resp.release() return json.loads(data) @asyncio.coroutine def patch(self, url): resp = yield from self.client.post(url) data = yield from resp.read() yield from resp.release() return json.loads(data)
async def create_line_to_position_map( self, client_session: aiohttp.ClientSession ) -> MutableMapping[str, Dict[int, int]]: result = defaultdict(dict) # type: MutableMapping[str, Dict[int, int]] current_file = '' position = -1 right_line_number = -1 headers = { 'Accept': 'application/vnd.github.diff', } url = ('{api_url}/repos/' '{organization}/{repo}/pulls/{pr}'.format( api_url=GITHUB_API_URL, organization=self.organization, repo=self.repo, pr=self.pr)) async with client_session.get(url, headers=headers) as response: async for line in response.content: line = line.decode() file_match = FILE_START_REGEX.match(line) if file_match: current_file = file_match.groups()[0].strip() right_line_number = -1 position = -1 continue elif line.startswith(NEW_FILE_SECTION_START): current_file = '' if not current_file: continue position += 1 hunk_match = HUNK_REGEX.match(line) if hunk_match: right_line_number = int(hunk_match.groups()[0]) - 1 elif not line.startswith('-'): right_line_number += 1 result[current_file][right_line_number] = position return result
def create_line_to_position_map( self, client_session: aiohttp.ClientSession ) -> MutableMapping[str, Dict[int, int]]: result = defaultdict(dict) # type: MutableMapping[str, Dict[int, int]] current_file = '' position = -1 right_line_number = -1 headers = { 'Accept': 'application/vnd.github.diff', } url = ('https://api.github.com/repos/' '{organization}/{repo}/pulls/{pr}'.format( organization=self.organization, repo=self.repo, pr=self.pr)) response = yield from client_session.get(url, headers=headers) content = yield from response.text() for line in content.split('\n'): file_match = FILE_START_REGEX.match(line) if file_match: current_file = file_match.groups()[0] right_line_number = -1 position = -1 continue elif line.startswith(NEW_FILE_SECTION_START): current_file = '' if not current_file: continue position += 1 hunk_match = HUNK_REGEX.match(line) if hunk_match: right_line_number = int(hunk_match.groups()[0]) - 1 elif not line.startswith('-'): right_line_number += 1 result[current_file][right_line_number] = position return result
async def call(session: aiohttp.ClientSession, params: Dict[str, Any], *, identity: Optional[Dict[str, str]] = None, check: bool = False ) -> Dict[str, Any]: """An asyncio coroutine which makes a HTTP request to the Ninchat Call API using the third-party aiohttp package. If check is set, raises a ninchat.call.APIError on "error" reply event. """ data = lib.request_content(params, identity=identity) async with session.post(lib.url, data=data, headers=lib.request_headers) as r: if r.status != HTTPStatus.OK: r.raise_for_status() e = await r.json() if check: lib.check_event(e) return e
async def get_url( url: str, session: ClientSession, with_text=False, with_data=False, headers=None ): """Fetch url. This function ensures that only one request is sent to a domain at one point in time and that the same url is not queried more than once. Parameters ---------- url : str session: ClientSession with_text: bool with_data : bool headers: dict Returns ------- RequestResult """ o = urlparse(url) if len(o.netloc) == 0: return RequestResult(exception=f"Could not parse URL: {url}") async with domain_lock: if o.netloc not in domain_locks: domain_locks[o.netloc] = asyncio.Lock() lock = domain_locks[o.netloc] async with lock: if url not in response_cache: for _ in range(3): try: logging.debug(f"GET {url}") async with session.request( method="GET", url=url, ssl=False, headers=headers ) as response: status = response.status if with_text: try: text = await response.text() except: text = await response.read() response_cache[url] = RequestResult( status=status, text=text ) elif with_data: data = await response.read() response_cache[url] = RequestResult( status=status, text=data ) else: response_cache[url] = RequestResult(status=status) except asyncio.TimeoutError: response_cache[url] = RequestResult(exception=f"Timeout for: {url}") except Exception as e: logging.debug(f"Error for: {url} ({e})") response_cache[url] = RequestResult( exception=f"Exception {e} for: {url}" ) if RequestResult.exception is None: break await asyncio.sleep(15) else: logging.debug(f"Cached {url}") return response_cache[url]
async def update_token(self): parsed_url = urlparse(self.settings.get("api_url")) self.api_url = "{url.scheme}://{url.netloc}".format(url=parsed_url) LOGGER.debug("API URL OYD %s", self.api_url) client_id = self.settings.get("client_id") client_secret = self.settings.get("client_secret") grant_type = self.settings.get("grant_type", "client_credentials") scope = self.settings.get("scope") if self.api_url is None: raise PDSError("Please configure the plugin, api_url is empty") if client_id is None: raise PDSError("Please configure the plugin, client_id is empty") if client_secret is None: raise PDSError( "Please configure the plugin, client_secret is empty") async with ClientSession() as session: body = { "client_id": client_id, "client_secret": client_secret, "grant_type": grant_type, } if scope is not None: body["scope"] = scope result = await session.post( self.api_url + "/oauth/token", json=body, ) result = await unpack_response(result) token = json.loads(result) self.token = token self.token_timestamp = time.time() LOGGER.info("update token: %s", self.token) """ Download the usage policy """ url = f"{self.api_url}/api/meta/usage" async with ClientSession() as session: result = await session.get( url, headers={ "Authorization": "Bearer " + self.token["access_token"] }, ) result = await unpack_response(result) self.settings["usage_policy"] = result LOGGER.debug("Usage policy %s", self.settings["usage_policy"]) """ Upload usage_policy as oca_schema_chunk """ async with ClientSession() as session: result = await session.post( "https://governance.ownyourdata.eu/api/usage-policy/parse", headers={ "Authorization": "Bearer " + self.token["access_token"] }, json={"ttl": self.settings["usage_policy"]}, ) result = await unpack_response(result) result = json.loads(result) result, err = map_parsed_usage_policy( result, cached_schema_to_map_against) await self.save( result, { "table": "tda.oca_chunks.H5F2YgEbXpSZjcNqAYevfGPFXSWUV1d2PnVg2ubkkKb" }, addition_meta={"missing": err}, )
async def start(self): """Start the outbound transport.""" self.client_session = ClientSession(cookie_jar=DummyCookieJar(), trust_env=True) return self
async def get_connection(url): s = ClientSession() sessions.append(s) conn = await s.get(url) dot() return conn
async def fetch_json(client_session: ClientSession, url) -> dict: async with client_session.get(url) as response: return await response.json()
async def get_bytes(url, headers=None): async with ClientSession() as asyncSession: async with asyncSession.get(url, headers=headers) as response: a = await response.read() return a
result = await resp.read() print('order: {:d}, result: {:s}'.format(order, str(result))) def request_page(order, session, loop): co = _request_page(order, session, loop) asyncio.run_coroutine_threadsafe(co, loop) try: loop = asyncio.get_event_loop() loopThread = threading.Thread(target=keep_loop, args=(loop, )) loopThread.setDaemon(True) loopThread.start() session = ClientSession(loop=loop) for i in range(5): loop.call_soon_threadsafe(request_page, i, session, loop) ## loop.call_soon()会卡住 ## loop.call_soon(request_page, i, session, loop) ## join()时ctrl-c无法结束, 而while True可以. ## loopThread.join() while True: time.sleep(1) except KeyboardInterrupt as e: ## 以下两行可以注释掉, 不会报错 asyncio.run_coroutine_threadsafe(session.close(), loop) time.sleep(1)
def session(self) -> ClientSession: if self._session is None: self._session = ClientSession(loop=self.loop) return self._session
def __init__(self): trace_config = TraceConfig() trace_config.on_request_start.append(self.on_request_start) trace_config.on_request_end.append(self.on_request_end) self.session = ClientSession(trace_configs=[trace_config]) self.logger = AppLogger()
async def fetch(session: aiohttp.ClientSession, url: Address) -> Json: async with session.get(url) as response: output = await response.json() # turning this into an async iterator instead of just returning the json gives us more chances to switch tasks return output
async def post_url(url: str, headers: dict = None) -> Tuple[ClientResponse, dict]: headers = headers or get_headers() async with ClientSession(headers=headers) as session: async with timeout(10): async with session.post(url) as resp: return await _extract_response_and_json_from_request(resp)
async def fetch_data(data): async with ClientSession() as session: url_data = url.format(data) async with session.get(url_data) as resp: data = json.loads(await resp.text()) return data['data']
while True: if len(healthy) > 90: healthy.remove(random.choice(healthy)) random.shuffle(healthy) tasks = [] for s in healthy: url = f'{base}/api/health/main/{s}' task = asyncio.ensure_future(bound_post(sem, url, session)) await asyncio.sleep(random.random() / 4) tasks.append(task) await asyncio.gather(*tasks) async def run_demo(): while True: shard = random.randint(0, shards - 1) state = random.randint(0, 6) url = f'{base}/api/status/main/{shard}/{state}' del url async def main(session): await run_init(session) await run_healthy(session) loop = asyncio.get_event_loop() with ClientSession() as session: loop.run_until_complete(main(session))
from aiohttp import ClientSession, web from aiohttp.web import HTTPBadRequest, HTTPUnsupportedMediaType import asyncio import logging from lib.metrics import REQUEST_TIME from lib.scoring import score from lib.util import fetch from prometheus_async.aio import time import uvloop logger = logging.getLogger('nsfwoid/score') asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) session = ClientSession() class Score(web.View): @time(REQUEST_TIME) async def post(self): request = self.request data = await request.post() try: url = data["url"] logger.info("Processing score request for URL: {0}".format(url)) image = await fetch(session, url) nsfw_prob = await score(image) return web.Response(text=nsfw_prob.astype(str)) except KeyError: raise HTTPBadRequest(text="Missing `url` POST parameter") except OSError as e: if "cannot identify" in str(e):
class Mushaf(commands.Cog): def __init__(self, bot): self.bot = bot self.session = ClientSession(loop=bot.loop) async def _mushaf(self, ctx, ref, show_tajweed): reference = QuranReference(ref) async with self.session.get( f'https://api.alquran.cloud/ayah/{reference.surah}:{reference.ayat_list}' ) as resp: if resp.status != 200: return await ctx.send(INVALID_VERSE) data = await resp.json() page = data['data']['page'] formatted_page = str(page).zfill(3) if show_tajweed: url = f'https://www.searchtruth.org/quran/images1/{formatted_page}.jpg' else: url = f'https://www.searchtruth.org/quran/images2/large/page-{formatted_page}.jpeg' arabic_page_number = convert_to_arabic_number(str(page)) em = discord.Embed(title=f'Page {page}' f'\n الصفحة{arabic_page_number}', colour=0x006400) em.set_author(name=f'Mushaf / مصحف', icon_url=ICON) em.set_image(url=url) await ctx.send(embed=em) @commands.command(name="mushaf") async def mushaf(self, ctx, ref: str, tajweed: bool = False): await self._mushaf(ctx, ref, tajweed) @mushaf.error async def on_mushaf_error(self, ctx, error): if isinstance(error, MissingRequiredArgument): await ctx.send(INVALID_INPUT.format(ctx.prefix)) @cog_ext.cog_slash( name="mushaf", description="View an ayah on the mushaf.", options=[ create_option( name="reference", description="The verse show on the mushaf, e.g. 1:4 or 2:255.", option_type=3, required=True), create_option( name="show_tajweed", description= "Should the mushaf highlight where tajweed rules apply?", option_type=5, required=False) ]) async def slash_mushaf(self, ctx: SlashContext, reference: str, show_tajweed: bool = False): await ctx.respond() await self._mushaf(ctx, reference, show_tajweed)
class upnp: def __init__(self, host, session: Optional[ClientSession] = None): self.host = host self.mute = False self.volume = 0 if session: self._session = session self._managed_session = False else: self._session = ClientSession() self._managed_session = True def __enter__(self): return self async def _SOAPrequest(self, action, arguments, protocole): headers = { "SOAPAction": '"urn:schemas-upnp-org:service:{protocole}:1#{action}"'.format( action=action, protocole=protocole), "content-type": "text/xml", } body = """<?xml version="1.0" encoding="utf-8"?> <s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"> <s:Body> <u:{action} xmlns:u="urn:schemas-upnp-org:service:{protocole}:1"> <InstanceID>0</InstanceID> {arguments} </u:{action}> </s:Body> </s:Envelope>""".format(action=action, arguments=arguments, protocole=protocole) response = None try: with timeout(DEFAULT_TIMEOUT): async with self._session.post( f"http://{self.host}:9197/upnp/control/{protocole}1", headers=headers, data=body, raise_for_status=True, ) as resp: response = await resp.content.read() except: pass return response async def async_get_volume(self): response = await self._SOAPrequest("GetVolume", "<Channel>Master</Channel>", "RenderingControl") if response is not None: volume_xml = response.decode("utf8") tree = ET.fromstring(volume_xml) for elem in tree.iter(tag="CurrentVolume"): self.volume = elem.text return self.volume async def async_set_volume(self, volume): await self._SOAPrequest( "SetVolume", "<Channel>Master</Channel><DesiredVolume>{}</DesiredVolume>". format(volume), "RenderingControl", ) async def async_get_mute(self): response = await self._SOAPrequest("GetMute", "<Channel>Master</Channel>", "RenderingControl") if response is not None: # mute_xml = response.decode('utf8') tree = ET.fromstring(response.decode("utf8")) mute = 0 for elem in tree.iter(tag="CurrentMute"): mute = elem.text if int(mute) == 0: self.mute = False else: self.mute = True return self.mute async def async_set_current_media(self, url): """ Set media to playback and play it.""" try: await self._SOAPrequest( "SetAVTransportURI", "<CurrentURI>{url}</CurrentURI><CurrentURIMetaData></CurrentURIMetaData>" .format(url=url), "AVTransport", ) await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport") except Exception: pass async def async_play(self): """ Play media that was already set as current.""" try: await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport") except Exception: pass
class Client(object): def __init__(self, url, dumper=None, loop=None): self.url = url self.dumper = dumper if not loop: loop = asyncio.get_event_loop() if not self.dumper: self.dumper = json.dumps self.client = ClientSession( loop=loop, headers={'content-type': 'application/json'}) def __del__(self): self.client.close() def __encode(self, method, params=None, id=None): try: data = self.dumper({ "jsonrpc": "2.0", "id": id, "method": method, "params": params }) except Exception as e: raise Exception("Can not encode: {}".format(e)) return data @asyncio.coroutine def call(self, method, params=None, id=None, schem=None): if not id: id = uuid4().hex try: resp = yield from self.client.post( self.url, data=self.__encode(method, params, id)) except Exception as err: raise Exception(err) if 200 != resp.status: raise InvalidResponse( "Error, server retunrned: {status}".format(status=resp.status)) try: data = yield from resp.json() except Exception as err: raise InvalidResponse(err) try: validate(data, ERR_JSONRPC20) return Response(**data) except ValidationError: # Passing data to validate response. # Good if does not valid to ERR_JSONRPC20 object. pass except Exception as err: raise InvalidResponse(err) try: validate(data, RSP_JSONRPC20) if id != data['id']: raise InvalidResponse( "Rsponse id {local} not equal {remote}".format( local=id, remote=data['id'])) except Exception as err: raise InvalidResponse(err) if schem: try: validate(data['result'], schem) except ValidationError as err: raise InvalidResponse(err) except Exception as err: raise InternalError(err) return Response(**data)
async def main(): connector = TCPConnector(limit=None) async with ClientSession(connector=connector) as session: await run(session, int(sys.argv[1]))
async def draw_profile(user: Member, session: ClientSession) -> Image: name_fnt = ImageFont.truetype(font_heavy_file, 30) name_u_fnt = ImageFont.truetype(font_unicode_file, 30) title_fnt = ImageFont.truetype(font_heavy_file, 22) title_u_fnt = ImageFont.truetype(font_unicode_file, 23) label_fnt = ImageFont.truetype(font_bold_file, 18) exp_fnt = ImageFont.truetype(font_bold_file, 13) large_fnt = ImageFont.truetype(font_thin_file, 33) rep_fnt = ImageFont.truetype(font_heavy_file, 26) rep_u_fnt = ImageFont.truetype(font_unicode_file, 30) text_fnt = ImageFont.truetype(font_file, 14) text_u_fnt = ImageFont.truetype(font_unicode_file, 14) symbol_u_fnt = ImageFont.truetype(font_unicode_file, 15) def _write_unicode(text, init_x, y, font, unicode_font, fill): write_pos = init_x for char in text: if char.isalnum() or char in string.punctuation or char in string.whitespace: draw.text((write_pos, y), char, font=font, fill=fill) write_pos += font.getsize(char)[0] else: draw.text((write_pos, y), u"{}".format(char), font=unicode_font, fill=fill) write_pos += unicode_font.getsize(char)[0] user_info = db.user(user) # get urls bg_url = await user_info.profile_background() profile_url = user.avatar_url async with session.get(bg_url) as r: bg_image = Image.open(BytesIO(await r.content.read())).convert("RGBA") async with session.get(profile_url) as r: profile_image = Image.open(BytesIO(await r.content.read())).convert("RGBA") # COLORS white_color = (240, 240, 240, 255) rep_fill = await user_info.rep_color() badge_fill = await user_info.badge_col_color() info_fill = await user_info.profile_info_color() info_fill_tx = (*info_fill[:3], 150) exp_fill = await user_info.profile_exp_color() title = await user_info.title() info = await user_info.info() total_exp = await user_info.total_exp() badges = await user_info.badges() rep = await user_info.rep() if badge_fill == (128, 151, 165, 230): level_fill = white_color else: level_fill = _contrast(exp_fill, rep_fill, badge_fill) # set canvas bg_color = (255, 255, 255, 0) result = Image.new("RGBA", (340, 390), bg_color) process = Image.new("RGBA", (340, 390), bg_color) # draw draw = ImageDraw.Draw(process) # puts in background bg_image = bg_image.resize((340, 340), Image.ANTIALIAS) bg_image = bg_image.crop((0, 0, 340, 305)) result.paste(bg_image, (0, 0)) # draw filter draw.rectangle([(0, 0), (340, 340)], fill=(0, 0, 0, 10)) draw.rectangle([(0, 134), (340, 325)], fill=info_fill_tx) # general content # draw profile circle multiplier = 8 lvl_circle_dia = 116 circle_left = 14 circle_top = 48 raw_length = lvl_circle_dia * multiplier # create mask mask = Image.new("L", (raw_length, raw_length), 0) draw_thumb = ImageDraw.Draw(mask) draw_thumb.ellipse((0, 0) + (raw_length, raw_length), fill=255, outline=0) # border lvl_circle = Image.new("RGBA", (raw_length, raw_length)) draw_lvl_circle = ImageDraw.Draw(lvl_circle) draw_lvl_circle.ellipse( [0, 0, raw_length, raw_length], fill=(255, 255, 255, 255), outline=(255, 255, 255, 250) ) # put border lvl_circle = lvl_circle.resize((lvl_circle_dia, lvl_circle_dia), Image.ANTIALIAS) lvl_bar_mask = mask.resize((lvl_circle_dia, lvl_circle_dia), Image.ANTIALIAS) process.paste(lvl_circle, (circle_left, circle_top), lvl_bar_mask) # put in profile picture total_gap = 6 border = int(total_gap / 2) profile_size = lvl_circle_dia - total_gap mask = mask.resize((profile_size, profile_size), Image.ANTIALIAS) profile_image = profile_image.resize((profile_size, profile_size), Image.ANTIALIAS) process.paste(profile_image, (circle_left + border, circle_top + border), mask) # write label text white_color = (240, 240, 240, 255) light_color = (160, 160, 160, 255) dark_color = (35, 35, 35, 255) head_align = 140 # determine info text color info_text_color = _contrast(info_fill, white_color, dark_color) _write_unicode( _truncate_text(user.name, 22).upper(), head_align, 142, name_fnt, name_u_fnt, info_text_color, ) # NAME _write_unicode( title.upper(), head_align, 170, title_fnt, title_u_fnt, info_text_color ) # draw divider draw.rectangle([(0, 323), (340, 324)], fill=(0, 0, 0, 255)) # box # draw text box draw.rectangle( [(0, 324), (340, 390)], fill=(info_fill[0], info_fill[1], info_fill[2], 255) ) # box _write_unicode("❤", 257, 9, rep_fnt, rep_u_fnt, info_text_color) draw.text( (_center(278, 340, str(rep), rep_fnt), 10), str(rep), font=rep_fnt, fill=info_text_color, ) # Exp Text label_align = 362 # vertical draw.text( (_center(0, 140, " RANK", label_fnt), label_align), " RANK", font=label_fnt, fill=info_text_color, ) # Rank draw.text( (_center(0, 340, " LEVEL", label_fnt), label_align), " LEVEL", font=label_fnt, fill=info_text_color, ) # Exp draw.text( (_center(200, 340, "BALANCE", label_fnt), label_align), "BALANCE", font=label_fnt, fill=info_text_color, ) # Credits if "linux" in platform.system().lower(): global_symbol = u"\U0001F30E " _write_unicode( global_symbol, 36, label_align + 5, label_fnt, symbol_u_fnt, info_text_color ) # Symbol _write_unicode( global_symbol, 134, label_align + 5, label_fnt, symbol_u_fnt, info_text_color ) # Symbol # userinfo global_rank = f"#{await _find_global_rank(user)}" global_level = str(_find_level(total_exp)) draw.text( (_center(0, 140, global_rank, large_fnt), label_align - 27), global_rank, font=large_fnt, fill=info_text_color, ) # Rank draw.text( (_center(0, 340, global_level, large_fnt), label_align - 27), global_level, font=large_fnt, fill=info_text_color, ) # Exp # draw level bar exp_font_color = _contrast(exp_fill, light_color, dark_color) exp_frac = int(total_exp - _level_exp(int(global_level))) exp_total = _required_exp(int(global_level) + 1) bar_length = int(exp_frac / exp_total * 340) draw.rectangle( [(0, 305), (340, 323)], fill=(level_fill[0], level_fill[1], level_fill[2], 245) ) # level box draw.rectangle( [(0, 305), (bar_length, 323)], fill=(exp_fill[0], exp_fill[1], exp_fill[2], 255) ) # box exp_text = "{}/{}".format(exp_frac, exp_total) # Exp draw.text( (_center(0, 340, exp_text, exp_fnt), 305), exp_text, font=exp_fnt, fill=exp_font_color, ) # Exp Text credit_txt = f"${await bank.get_balance(user)}" draw.text( (_center(200, 340, credit_txt, large_fnt), label_align - 27), _truncate_text(credit_txt, 18), font=large_fnt, fill=info_text_color, ) # Credits if title == "": offset = 170 else: offset = 195 margin = 140 txt_color = _contrast(info_fill, white_color, dark_color) for line in textwrap.wrap(info, width=32): _write_unicode(line, margin, offset, text_fnt, text_u_fnt, txt_color) offset += text_fnt.getsize(line)[1] + 2 # sort badges priority_badges = [] for badgename in badges.keys(): badge = badges[badgename] priority_num = badge["priority_num"] if priority_num != 0 and priority_num != -1: priority_badges.append((badge, priority_num)) sorted_badges = sorted(priority_badges, key=operator.itemgetter(1), reverse=True) # TODO: simplify this. it shouldn't be this complicated... sacrifices conciseness for customizability if (await db.badge_type()) == "circles": # circles require antialiasing vert_pos = 172 right_shift = 0 left = 9 + right_shift size = 38 total_gap = 4 # /2 hor_gap = 6 vert_gap = 6 border_width = int(total_gap / 2) multiplier = 6 # for antialiasing raw_length = size * multiplier mult = [(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1), (0, 2), (1, 2), (2, 2)] for num in range(9): coord = ( left + int(mult[num][0]) * int(hor_gap + size), vert_pos + int(mult[num][1]) * int(vert_gap + size), ) if num < len(sorted_badges[:9]): pair = sorted_badges[num] badge = pair[0] bg_color = badge["bg_img"] border_color = badge["border_color"] # draw mask circle mask = Image.new("L", (raw_length, raw_length), 0) draw_thumb = ImageDraw.Draw(mask) draw_thumb.ellipse((0, 0) + (raw_length, raw_length), fill=255, outline=0) # determine image or color for badge bg if await self._valid_image_url(bg_color): # get image async with session.get(bg_color) as r: badge_image = Image.open(BytesIO(await r.content.read())).convert("RGBA") badge_image = badge_image.resize((raw_length, raw_length), Image.ANTIALIAS) # structured like this because if border = 0, still leaves outline. if border_color: square = Image.new("RGBA", (raw_length, raw_length), border_color) # put border on ellipse/circle output = ImageOps.fit( square, (raw_length, raw_length), centering=(0.5, 0.5) ) output = output.resize((size, size), Image.ANTIALIAS) outer_mask = mask.resize((size, size), Image.ANTIALIAS) process.paste(output, coord, outer_mask) # put on ellipse/circle output = ImageOps.fit( badge_image, (raw_length, raw_length), centering=(0.5, 0.5) ) output = output.resize( (size - total_gap, size - total_gap), Image.ANTIALIAS ) inner_mask = mask.resize( (size - total_gap, size - total_gap), Image.ANTIALIAS ) process.paste( output, (coord[0] + border_width, coord[1] + border_width), inner_mask, ) else: # put on ellipse/circle output = ImageOps.fit( badge_image, (raw_length, raw_length), centering=(0.5, 0.5) ) output = output.resize((size, size), Image.ANTIALIAS) outer_mask = mask.resize((size, size), Image.ANTIALIAS) process.paste(output, coord, outer_mask) else: plus_fill = exp_fill # put on ellipse/circle plus_square = Image.new("RGBA", (raw_length, raw_length)) plus_draw = ImageDraw.Draw(plus_square) plus_draw.rectangle( [(0, 0), (raw_length, raw_length)], fill=(info_fill[0], info_fill[1], info_fill[2], 245), ) # draw plus signs margin = 60 thickness = 40 v_left = int(raw_length / 2 - thickness / 2) v_right = v_left + thickness v_top = margin v_bottom = raw_length - margin plus_draw.rectangle( [(v_left, v_top), (v_right, v_bottom)], fill=(plus_fill[0], plus_fill[1], plus_fill[2], 245), ) h_left = margin h_right = raw_length - margin h_top = int(raw_length / 2 - thickness / 2) h_bottom = h_top + thickness plus_draw.rectangle( [(h_left, h_top), (h_right, h_bottom)], fill=(plus_fill[0], plus_fill[1], plus_fill[2], 245), ) # put border on ellipse/circle output = ImageOps.fit( plus_square, (raw_length, raw_length), centering=(0.5, 0.5) ) output = output.resize((size, size), Image.ANTIALIAS) outer_mask = mask.resize((size, size), Image.ANTIALIAS) process.paste(output, coord, outer_mask) result = Image.alpha_composite(result, process) result = _add_corners(result, 25) return result
async def generate_requests() -> list: urls = mapUrls.keys() async with ClientSession(connector=TCPConnector( verify_ssl=False)) as session: responses = await fetch_all(session, urls) return responses
async def run(args: argparse): """ Metodo que realizada una peticion asincrona post por cada elemento de los diccionarios proporcionados :param args: :return: """ name_username: str name_password: str my_json: Dict[str, str] = json.loads(args.json) keys_json = list(my_json) # si el json tiene 1 valor se entiende que es la pass, si tiene 2 hay user y pass if len(keys_json) >= 2: name_username = keys_json[0] name_password = keys_json[1] elif len(keys_json) == 1: name_username = '' name_password = keys_json[0] else: logger.critical('json is not valid') sys.exit(1) dict_users: List = list() dict_passwords: List = list() # creamos un diccionaro de users if args.Login is None: dict_users.append(args.login) else: file_username: Path = Path(args.Login) if file_username.exists(): dict_users = file_username.read_text().split('\n') dict_users.remove('') # eliminar lineas en blanco else: logger.critical(f'File "{args.Login}" not exits') sys.exit(1) # creamos un diccionaro de passwords if args.Password is None: dict_passwords.append(args.password) else: file_password: Path = Path(args.Password) if file_password.exists(): dict_passwords = file_password.read_text().split('\n') else: logger.critical(f'File "{args.Password}" not exits') sys.exit(1) is_online(args.target) len_user = len(dict_users) len_pwd = len(dict_passwords) user: str pwd: str cont: int = 1 tasks: List = list() # create instance of Semaphore sem: asyncio.locks.Semaphore = asyncio.Semaphore(1000) # Create client session that will ensure we dont open new connection per each request. async with ClientSession() as session: for user in dict_users: # if len(user) == 0: # si existe alguna liena en blanco la omitimos, ya se hace al crear la lista # continue # Actualizamos el username si es necesario ponerlo, ya se hace al crear la lista if name_username in my_json.keys(): my_json[name_username] = user for pwd in dict_passwords: # Actualizamos la contraseña if name_password in my_json.keys(): my_json[name_password] = pwd # pass Semaphore and session to every GET request update_json = json.dumps( my_json ) # si envio el json como dict llega vacio a la funcion (i dont known) task = asyncio.ensure_future( bound_fetch(sem, session, args, user, pwd, update_json, (len_user * len_pwd), cont)) tasks.append(task) cont += 1 responses = asyncio.gather(*tasks) await responses
def __init__(self, bot): self.bot = bot self.session = ClientSession(loop=bot.loop)
async def __create_aiohttp_client_session(self): if not self._aiohttp_client_session: ck = CookieJar(unsafe=True) self._aiohttp_client_session = ClientSession(cookie_jar=ck)
class Database(): ins = None init = False def __new__(cls, endpoint:str=None) -> Any: if not cls.ins: cls.ins = super().__new__(cls) return cls.ins def __init__(self, endpoint:str=None) -> None: if Database.init: return Database.init = True self.sesison = ClientSession() # TOOD: do timeout self.endpoint = endpoint log.debug(f"endpoint set to {self.endpoint}") async def perform(self, method:str, endpoint:str, *, params:dict=None, json:dict=None) -> Optional[dict]: try: async with self.sesison.request(method, f"{self.endpoint}{endpoint}", params=params, json=json) as resp: payload = await resp.json(content_type=None) log.debug(f"{method} {endpoint} (params: {params}, json: {json}) -> {resp.status}") if resp.status == 200: return payload else: raise DatabaseError() except DatabaseError as e: raise e except Exception as e: log.error(f"Further information: ", exc_info=True) raise e get = partialmethod(perform, "GET") post = partialmethod(perform, "POST") delete = partialmethod(perform, "DELETE") async def get_playlists(self) -> Optional[dict]: return await self.get("/playlist") async def get_playlist(self, name:str, *, limit:int=1000) -> Optional[dict]: return await self.get(f"/playlist/{name}", params={"limit": limit}) async def create_playlist(self, name:str) -> None: await self.post("/playlist", params={"name": name}) async def delete_playlist(self, name:str) -> None: await self.delete(f"/playlist", params={"name": name}) async def add_to_playlist(self, name:str, uris:Sequence[str]) -> None: await self.post(f"/playlist/{name}", json={"entries": uris}) async def delete_from_playlist(self, name:str, uri:str) -> None: await self.delete(f"/playlist/{name}", params={"uri": uri}) async def get_likes(self, limit:int=1000) -> Optional[dict]: return await self.get("/likes", params={"limit": limit}) async def toggle_like(self, uri:str, like:bool) -> None: await self.post("/likes", params={"uri": uri, "like": 1 if like else 0}) async def is_liked(self, uri:str) -> Optional[dict]: return await self.get("/likes/resolve", params={"uri": uri}) async def get_cache(self, uri:str) -> Optional[dict]: return await self.get("/cache", params={"uri": uri}) async def store_cache(self, entries:Sequence[dict]) -> None: await self.post("/cache", json={"entries": entries}) async def update_gpm(self, entries:Sequence[dict], user:str) -> None: await self.post("/gpm/update", params={"name": user}, json={"entries": entries}) async def search_gpm(self, query:str, *, limit=100) -> Optional[dict]: return await self.get("/gpm/search", params={"limit": limit, "query": query}) async def resolve_gpm(self, uri:str) -> Optional[dict]: return await self.get("/gpm", params={"uri": uri})
async def _get_async_session(endpoint_uri: URI) -> ClientSession: cache_key = generate_cache_key(endpoint_uri) if cache_key not in _async_session_cache: await cache_async_session(endpoint_uri, ClientSession(raise_for_status=True)) return _async_session_cache.get_cache_entry(cache_key)
async def _get(url: str, session: aiohttp.ClientSession): async with session.get(url, headers=_HEADERS) as response: assert response.status == 200 return await response.json(loads=ujson.loads)
async def fetch_playlist(playlistId: str, chunk_size: int = 1024): """ Fetch all metadata of a YouTube playlist. Performs multiple asynchronous HTTP requests to obtain all metadata and playlist items. Does not provide rich YouTube metadata: fields that are included in browse_request queries for modern JS YT-Clients. Result: { 'id': string, 'title': string, 'description': string, 'thumbnail': string, 'length': integer, 'views': integer, 'uploader': { 'name': string, 'url': string }, 'items': [ { 'id': string, 'title': string, 'uploader': { 'name': string, 'url': string }, 'lengthSeconds': integer, } ] } """ # # Initialize the result # playlist = {'id': playlistId, 'items': []} # # Build the headers for the ClientSession. # headers = get_default_headers() # Only accept HTML. headers['Accept'] = 'text/html' # # Retrieve landing page. # # Open a auto-raising ClientSession with default cookie handling. async with ClientSession(headers=headers, raise_for_status=True) as session: # Step 1: Get the initial landing page. async with session.get('https://www.youtube.com/playlist', params={'list': playlistId}) as response: # Assert that this really worked the way we wanted. assert response.status == 200 assert response.content_type == 'text/html' encoding = response.get_encoding() assert is_valid_encoding(encoding) # Retrieve the '//div[@id=""]' node. is_content = lambda x: (x.tag, x.get('id')) == ('div', '') parser = etree.HTMLPullParser(events=('start', 'end')) content, discard = None, True while content is None and not response.content.at_eof(): # Feed the parser the next chunk of data. parser.feed( (await response.content.read(chunk_size)).decode(encoding)) for event, node in parser.read_events(): if event == 'start': if is_content(node): # Content node reached, stop discarding. discard = False continue if is_content(node): # Content node finished, exit. content = node break if discard: # Discard everything before this point. node.clear() for ancestor in node.xpath('ancestor-or-self::*'): while ancestor.getprevious() is not None: del ancestor.getparent()[0] # # Parse the playlist header. # pl_header = content[0] assert pl_header.get('id') == 'pl-header' # Get the thumbnail. pl_thumb = pl_header[0][0] assert pl_thumb.tag == 'img' playlist['thumbnail'] = pl_thumb.get('src') # Get the title. pl_title = pl_header[1][0] assert pl_title.tag == 'h1' playlist['title'] = pl_title.text.strip() # Get the uploader. pl_uploader = pl_header[1][1][0][0] assert pl_uploader.tag == 'a' playlist['uploader'] = { 'name': pl_uploader.text, 'url': 'https://www.youtube.com' + pl_uploader.get('href') } # Get the length. pl_length = pl_header[1][1][1] assert pl_length.tag == 'li' playlist['length'] = parse_int(pl_length.text, aggressive=True) # Get the view count. pl_views = pl_header[1][1][2] assert pl_views.tag == 'li' playlist['views'] = parse_int(pl_views.text, aggressive=True) # Get the description. pl_description = pl_header[1][2][0] assert pl_description.tag == 'span' playlist['description'] = pl_description.text.strip() # # Parse the playlist items. # def parse_item(node): item = {'id': node.get('data-video-id')} # Get the video thumbnail. vid_thumb = node[2][0][0][0][0][0][0] assert vid_thumb.tag == 'img' item['thumbnail'] = vid_thumb.get('data-thumb') # Get video title. vid_title = node[3][0] assert vid_title.tag == 'a' item['title'] = vid_title.text.strip() # Get video uploader. vid_uploader = node[3][1][0] assert vid_uploader.tag == 'a' item['uploader'] = { 'name': vid_uploader.text, 'url': 'https://www.youtube.com' + vid_uploader.get('href') } # Get video length. vid_length = node[6][0][0][0] assert vid_length.tag == 'span' item['lengthSeconds'] = parse_ts(vid_length.text) return item pl_items = content[1][0][0][0][0] assert pl_items.get('id') == 'pl-load-more-destination' playlist['items'] += map(parse_item, pl_items) # # Fetch and parse all continuations. # load_more = index_s(content, 1, 0, 0, 1) assert load_more.tag == 'button' load_more = load_more.get( 'data-uix-load-more-href') if load_more is not None else None while load_more is not None: # Request the continuation contents. async with session.get('https://www.youtube.com' + load_more, headers={'Accept': 'application/json'}) as response: # Assert that this really worked the way we wanted. assert response.status == 200 assert response.content_type == 'application/json' encoding = response.get_encoding() assert is_valid_encoding(encoding) # Parse the result data (large!) # TODO: This ought to be streamed as well. data = await response.json() assert 'content_html' in data assert 'load_more_widget_html' in data # Parse all new items. parser = etree.HTMLPullParser(events=('end', )) parser.feed(data['content_html']) for _, node in parser.read_events(): if node.tag == 'tr': playlist['items'] += [parse_item(node)] # Discard everything before this point. node.clear() for ancestor in node.xpath('ancestor-or-self::*'): while ancestor.getprevious() is not None: del ancestor.getparent()[0] # Extract the next continuation link match = re.search(r'data-uix-load-more-href=\"(.+?)\"', data['load_more_widget_html']) if match: # Next continuation link. load_more = match.group(1) else: # No more continuations. load_more = None return playlist
def __init__(self, username, password, timeout=10): self._loop = asyncio.get_event_loop() self._session = ClientSession(loop=self._loop, auth=BasicAuth(username, password)) self._timeout = timeout
async def get_json_content(url): async with ClientSession() as session: async with session.get(url) as resp: print("Response:" + str(resp.status)) json_string = await resp.text() return loads(json_string)
async def _auth_registry_request(url: URL, method: str, auth_headers: Dict, session: ClientSession) -> Tuple[Dict, Dict]: if not config.REGISTRY_AUTH or not config.REGISTRY_USER or not config.REGISTRY_PW: raise exceptions.RegistryConnectionError( "Wrong configuration: Authentication to registry is needed!") # auth issue let's try some authentication get the auth type auth_type = None auth_details: Dict[str, str] = {} for key in auth_headers: if str(key).lower() == "www-authenticate": auth_type, auth_value = str(auth_headers[key]).split(" ", 1) auth_details = { x.split("=")[0]: x.split("=")[1].strip('"') for x in auth_value.split(",") } break if not auth_type: raise exceptions.RegistryConnectionError( "Unknown registry type: cannot deduce authentication method!") auth = BasicAuth(login=config.REGISTRY_USER, password=config.REGISTRY_PW) # bearer type, it needs a token with all communications if auth_type == "Bearer": # get the token token_url = URL(auth_details["realm"]).with_query( service=auth_details["service"], scope=auth_details["scope"]) async with session.get(token_url, auth=auth) as token_resp: if not token_resp.status == HTTPStatus.OK: raise exceptions.RegistryConnectionError( "Unknown error while authentifying with registry: {}". format(str(token_resp))) bearer_code = (await token_resp.json())["token"] headers = {"Authorization": "Bearer {}".format(bearer_code)} async with getattr(session, method.lower())(url, headers=headers) as resp_wtoken: if resp_wtoken.status == HTTPStatus.NOT_FOUND: logger.exception("path to registry not found: %s", url) raise exceptions.ServiceNotAvailableError(str(url)) if resp_wtoken.status > 399: logger.exception( "Unknown error while accessing with token authorized registry: %s", str(resp_wtoken), ) raise exceptions.RegistryConnectionError(str(resp_wtoken)) resp_data = await resp_wtoken.json(content_type=None) resp_headers = resp_wtoken.headers return (resp_data, resp_headers) elif auth_type == "Basic": # basic authentication should not be since we tried already... async with getattr(session, method.lower())(url, auth=auth) as resp_wbasic: if resp_wbasic.status == HTTPStatus.NOT_FOUND: logger.exception("path to registry not found: %s", url) raise exceptions.ServiceNotAvailableError(str(url)) if resp_wbasic.status > 399: logger.exception( "Unknown error while accessing with token authorized registry: %s", str(resp_wbasic), ) raise exceptions.RegistryConnectionError(str(resp_wbasic)) resp_data = await resp_wbasic.json(content_type=None) resp_headers = resp_wbasic.headers return (resp_data, resp_headers) raise exceptions.RegistryConnectionError( f"Unknown registry authentification type: {url}")
class SpectateMotor: def __init__(self, bot): self.bot = bot self.session = ClientSession(loop=self.bot.loop) self.map = None self.bots = [] self.turns = 0 self.latest = {} self.update_task = None async def interrupt_spectating(self): await self.map.delete() self.update_task.cancel() self.update_task = None def update_turns(self): self.turns += 1 def interrupt_variables(self): self.bots = [] self.turns = 0 self.latest = {} async def do_restart(self, ctx, reason): counter = await ctx.send( f':warning: | **Initialized restart due to {reason}**') try: await self.interrupt_spectating() self.interrupt_variables() except: pass async for m in self.bot.get_channel(505815417269518346).history( limit=1000): try: if m.id != counter.id: await m.delete() except: continue await asyncio.sleep(1) for count in range(5): await counter.edit( content=f':warning: | **Restarting in {abs(count - 5)}s**') await asyncio.sleep(1) await counter.delete() os.execl(sys.executable, sys.executable, *sys.argv) async def updater(self, channel): await self.bot.wait_until_ready() self.map = await self.bot.get_channel(channel).send('Loading...') while not self.bot.is_closed(): await asyncio.sleep(3) print(f'Getting map...\n----------') async with self.session.get( "https://league.travitia.xyz/map") as r: map = await r.json() if map == self.latest: continue self.latest = map out = "" try: for row in map: out = f"{out}\n{''.join(['█' if not i else i['name'][0] for i in row])}" print('Getting bots alive...\n----------') async with self.session.get( "https://league.travitia.xyz/bots") as bots: self.bots = self.bots = sorted((await bots.json()), key=lambda d: -d['hp']) except: print( f'Game end. Ended in {self.turns} Generations...\n----------' ) await self.map.edit( content= f"```fix\n---GAME ENDED---\n{map['text'].capitalize()}\nTotal Generations: {self.turns}\nThis Game Took: ~{datetime.timedelta(seconds=self.turns * 3)}```" ) return self.interrupt_variables() self.update_turns() hp = '\n'.join( [f"{bot['name']}: {bot['hp']} HP" for bot in self.bots]) await self.map.edit( content=f'```fix\n{out}\n\n{hp}\n\nGeneration {self.turns}```') @commands.command(aliases=['s', 'st', 'sp', 'spect']) async def spectate(self, ctx): """Spectate The Game in LIVE Mode""" _ = await ctx.send('Trying to connect... Check <#505815417269518346>') if self.update_task is None: self.update_task = self.bot.loop.create_task( self.updater(505815417269518346)) await _.edit( content="🔴 **NOW LIVE!** Check <#505815417269518346>") else: await _.edit( content='Already spectating... Check <#505815417269518346>') @commands.command(aliases=['stahp']) async def stop(self, ctx): """Stop Spectating The Game""" if self.update_task is None: await ctx.send('Not spectating...') else: try: await self.interrupt_spectating() self.interrupt_variables() except: pass await ctx.send('Stopped spectating...') @commands.command(aliases=['newgame', 'ng']) async def new(self, ctx): """Starts a New Game""" async with self.session.get("https://league.travitia.xyz/new") as req: if (await req.json())["status"] == "error": await ctx.send("Game already running... you can't start new..." ) else: await ctx.send("New game successfully created...")
def __init__(self, gcp_session: ClientSession(), token: [str], logging_context: LoggingContext): self.gcp_session = gcp_session self.token = token self.logging_context = logging_context
async def get_json_data(url: str) -> dict: async with ClientSession(connector=TCPConnector(ssl=False)) as session: async with session.get(url=url) as response: return await response.json()