async def fetch_head(endpoint, session: ClientSession = None): """ Querying the endpoint for the headers. """ try: async with session.head(endpoint) as response: return response, None, endpoint except (asyncio.TimeoutError, ClientConnectorError) as err: return None, err, endpoint
async def get_webmention_endpoint(session: ClientSession, target: URL) -> Optional[URL]: """ Given a target URL, find the webmention endpoint, if any. """ if target.scheme not in {"http", "https"}: return None async with session.head(target) as response: for rel, params in response.links.items(): if "webmention" in rel.split(" "): return target.join(URL(params["url"])) if "text/html" not in response.headers.get("content-type", ""): return None async with session.get(target) as response: try: html = await response.text() except UnicodeDecodeError: return None soup = BeautifulSoup(html, "html.parser") link = soup.find(rel="webmention", href=True) if link: return target.join(URL(link["href"])) return None
async def download_async(session: aiohttp.ClientSession, package_name: str, url: str): """ Async Download Function, catches most network errors and ignores them. `async def` makes an asynchronous function and returns a coroutine. The coroutine can be run in a task, `await`ed in another coroutine, or in `asyncio.run` as "main function". :param session: aiohttp.ClientSession :param package_name: str :param url: str :return: None """ try: # `async with` is almost the Python `with` statement, # but run the resource acquisition asynchronously async with session.head(url) as rsp: if rsp.status == 200: print(f'Download: Found {package_name} 200') exit(0) elif rsp.status == 404: print(f'Download: {package_name} not found') else: print( f'Download: {package_name} failed for reason: {rsp.status}' ) except aiohttp.ClientError: pass except asyncio.TimeoutError: pass
async def get_resource_extension(session: ClientSession, url: str) -> str: """ Return the extension of a remote image. """ async with session.head(url) as response: content_type = response.headers["content-type"] extension = mimetypes.guess_extension(content_type) return extension or ""
class Client: _client = None def __init__(self, loop, url=None): self._client = ClientSession(loop=loop) self._url = url @property def cli(self): return self._client async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_value, traceback): pass def handler_url(self, url): if url.startswith("http"): return url if self._url: return "{}{}".format(self._url, url) return url def request(self, method, url, *args, **kwargs): return self._client.request(method, self.handler_url(url), *args, **kwargs) def get(self, url, allow_redirects=True, **kwargs): return self._client.get(self.handler_url(url), allow_redirects=True, **kwargs) def post(self, url, data=None, **kwargs): return self._client.post(self.handler_url(url), data=data, **kwargs) def put(self, url, data=None, **kwargs): return self._client.put(self.handler_url(url), data=data, **kwargs) def delete(self, url, **kwargs): return self._client.delete(self.handler_url(url), **kwargs) def head(self, url, allow_redirects=False, **kwargs): return self._client.head(self.handler_url(url), allow_redirects=allow_redirects, **kwargs) def options(self, url, allow_redirects=True, **kwargs): return self._client.options(self.handler_url(url), allow_redirects=allow_redirects, **kwargs) def close(self): self._client.close()
async def _check(self, session: aiohttp.ClientSession, username: str) -> None: async with session.head( f'https://www.tiktok.com/@{username}') as response: if response.status == 200 and len(username) > 2: print('%s[UNAVAILABLE] https://www.tiktok.com/@%s%s' % ('\u001b[31;1m', username, '\u001b[0m')) else: print('%s[AVAILABLE] https://www.tiktok.com/@%s%s' % ('\u001b[32;1m', username, '\u001b[0m')) write_file(username)
class S3LogFileManager(LogFileManager): def __init__(self, endpoint_url, bucket_name="debian-janitor", trace_configs=None): import boto3 self.base_url = endpoint_url + ("/%s/" % bucket_name) self.session = ClientSession(trace_configs=trace_configs) self.s3 = boto3.resource("s3", endpoint_url=endpoint_url) self.s3_bucket = self.s3.Bucket(bucket_name) def _get_key(self, pkg, run_id, name): return "logs/%s/%s/%s.gz" % (pkg, run_id, name) def _get_url(self, pkg, run_id, name): return "%s%s" % (self.base_url, self._get_key(pkg, run_id, name)) async def has_log(self, pkg, run_id, name): url = self._get_url(pkg, run_id, name) async with self.session.head(url) as resp: if resp.status == 404: return False if resp.status == 200: return True if resp.status == 403: return False raise LogRetrievalError( "Unexpected response code %d: %s" % (resp.status, await resp.text()) ) async def get_log(self, pkg, run_id, name, timeout=10): url = self._get_url(pkg, run_id, name) client_timeout = ClientTimeout(timeout) async with self.session.get(url, timeout=client_timeout) as resp: if resp.status == 404: raise FileNotFoundError(name) if resp.status == 200: return BytesIO(gzip.decompress(await resp.read())) if resp.status == 403: raise PermissionError(await resp.text()) raise LogRetrievalError( "Unexpected response code %d: %s" % (resp.status, await resp.text()) ) async def import_log(self, pkg, run_id, orig_path, timeout=360): with open(orig_path, "rb") as f: data = gzip.compress(f.read()) key = self._get_key(pkg, run_id, os.path.basename(orig_path)) self.s3_bucket.put_object(Key=key, Body=data, ACL="public-read") async def delete_log(self, pkg, run_id, name): key = self._get_key(pkg, run_id, name) self.s3_bucket.delete_objects(Delete={"Objects": [{"Key": key}]})
async def load_metadata(self, session: ClientSession, verbose: bool): if not self.url: return try: if verbose: print(f"Getting content length for {self.url}") async with session.head(self.url) as resp: if resp.status >= 400: raise ValueError(f"Status={resp.status} for HEAD request") if 'Content-Length' in resp.headers: self.file_len = int(resp.headers['Content-Length']) except Exception as ex: print_err(f"Unable to load metadata for {self}: {ex}")
async def _inner_fetch(self, session: ClientSession, resp: UrlFetchResponse, urltarget: UrlTarget, timer: Timer) -> None: try: async with session.head(urltarget.url) as response: timer.stop() resp.status = response.status if is_onsite(urltarget): await self._do_get(session, resp, urltarget, timer) except aiohttp.ClientResponseError as e: # Fixes ScholliYT/Broken-Links-Crawler-Action#8 if e.status == 405: await self._do_get(session, resp, urltarget, timer) else: raise e
async def _check_url(self, url: str, session: aiohttp.ClientSession) -> UrlStatus: delay = self._host_manager.get_delay(url) await asyncio.sleep(delay) try: async with session.head(url, allow_redirects=True, ssl=self._ssl_context) as response: if _is_http_code_success(response.status): return await self._process_response(url, response) # if status != 200, fallback to get await asyncio.sleep(delay) async with session.get(url, allow_redirects=True) as response: return await self._process_response(url, response) except (KeyboardInterrupt, CancelledError): raise # pragma: no cover except Exception as e: return UrlStatus(False, classify_exception(e, url))
async def fetch_meta_by_year(session: aiohttp.ClientSession, year: int, url: URL) -> None: """ This function is only used in `fetch_meta` and relatively short. So I write it inside.""" self.logger.debug('Year %d | %s', year, url) try: async with session.head(f'{url}') as response: self.metadata[year]['total_pages'] = int( response.headers['X-Total-Pages']) self.metadata[year]['url'] = url self.logger.info( 'Year %d | Total pages %s | Total number of challenges %s', year, response.headers['X-Total-Pages'], response.headers['X-Total']) return int(response.headers['X-Total']) except aiohttp.ClientResponseError: logging.error('Year %d | Fetching failed', year) return 0
async def check(session: ClientSession, url: URL, redirect: bool = False) -> ClientResponse: """Check the status of a website.""" async with session.head(url, allow_redirects=redirect) as response: return response
async def get_timestamp_from_url(url: str, session: aiohttp.ClientSession): async with session.head(url, allow_redirects=True) as response: return _extract_timestamp(response.headers)
async def get_video_approx_size(session: ClientSession, url: str): try: async with session.head(url) as response: return int(response.headers["Content-Length"]) except: return 0
class Youtube: def __init__(self): self.backend = build(YOUTUBE_API_SERVICE_NAME, YOUTUBE_API_VERSION, developerKey=secrets.DEVELOPER_KEY) self.session = ClientSession() async def close(self): await self.session.close() async def load(self, method, num, **parameters): parameters['maxResults'] = 50 pageToken = '' while True: if 'noPageToken' not in parameters: parameters['pageToken'] = pageToken url = getattr(self.backend, method)().list(**{k: v for k, v in parameters.items() if k != 'noPageToken'}).uri async with self.session.get(url) as response: result = await response.json() if 'items' not in result: print(url, pageToken, response.status, file=sys.stderr) print(result, file=sys.stderr) for item in result['items']: yield item num -= 1 if num == 0: return if 'nextPageToken' not in result or len(result['items']) == 0: return pageToken = result['nextPageToken'] async def search(self, query, max_results=50): if type(query) == list: query = ' '.join(query) async for item in self.load('search', max_results, q=query, part='id,snippet', type='video', regionCode='US', safeSearch='strict', ): yield item async def video(self, videoIds, max_results=50): async for item in self.load('videos', max_results, part='id,snippet,contentDetails,player', id=','.join(videoIds), regionCode='US', ): yield item async def related(self, target, max_results=50): async for item in self.load('search', max_results, part='id,snippet', relatedToVideoId=target, type='video', safeSearch='strict', videoType='movie', regionCode='US', ): yield item async def categories(self, target, max_results=50): async for item in self.load('videoCategories', max_results, part='id,snippet', regionCode='US', noPageToken=True): yield item async def popular(self, category, max_results=50): async for item in self.load('search', max_results, part='id,snippet', type='video', videoCategoryId=category, order='viewCount', regionCode='US', q='', ): yield item async def random(self, _, max_results=50): query = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(3)) async for item in self.search(query, max_results): yield item async def exists(self, videoIds): async def test_existence(videoId): url = 'https://i.ytimg.com/vi/%s/default.jpg' % videoId async with self.session.head(url) as request: await request.text() return request.status == 200 if type(videoIds) == str: return await test_existence(videoIds) else: return await asyncio.gather(*[asyncio.create_task(test_existence(videoId)) for videoId in videoIds])
class AcmeClient: """ACME compliant client.""" FINALIZE_DELAY = 3.0 """The delay in seconds between finalization attemps.""" INVALID_NONCE_RETRIES = 5 """The number of times the client should retry when the server returns the error *badNonce*.""" def __init__( self, *, directory_url: str, private_key: str, contact: typing.Dict[str, str] = None, server_cert: str = None, kid: str = None, hmac_key: str = None, ): """Creates an :class:`AcmeClient` instance. :param directory_url: The ACME server's directory :param private_key: Path of the private key to use to register the ACME account. Must be a PEM-encoded RSA or EC key file. :param contact: :class:`dict` containing the contact info to supply on registration. May contain a key *phone* and a key *email*. :param server_cert: Path of the server certificate to add to the SSL context :param kid: The external account binding's key identifier to be used on registration :param hmac_key: The external account binding's symmetric encryption key to be used on registration """ self._ssl_context = ssl.create_default_context() if server_cert: # Add our self-signed server cert for testing purposes. self._ssl_context.load_verify_locations(cafile=server_cert) self._session = ClientSession( headers={"User-Agent": f"acmetk Client {__version__}"}) self._directory_url = directory_url self._private_key, self._alg = self._open_key(private_key) # Filter empty strings self._contact = {k: v for k, v in contact.items() if len(v) > 0} self._directory = dict() self._nonces = set() self._account = None self._challenge_solvers = dict() self.eab_credentials = (kid, hmac_key) @property def eab_credentials(self) -> ExternalAccountBindingCredentials: """The client's currently stored external account binding credentials Getter: Returns the client's currently stored external account binding credentials to be used on registration. Setter: Sets the client's stored external account binding credentials :param credentials: The kid and hmac_key :raises: :class:`ValueError` If the tuple does not contain exactly the kid and hmac_key. """ return self._eab_credentials @eab_credentials.setter def eab_credentials(self, credentials: typing.Tuple[str]): """Sets the client's stored external account binding credentials :param credentials: The kid and hmac_key :raises: :class:`ValueError` If the tuple does not contain exactly the kid and hmac_key. """ if isinstance(credentials, tuple) and len(credentials) == 2: self._eab_credentials = ExternalAccountBindingCredentials( *credentials) else: raise ValueError( "A tuple containing the kid and hmac_key is required") def _open_key(self, private_key): with open(private_key, "rb") as pem: data = pem.read() certs = acmetk.util.pem_split(data.decode()) if len(certs) != 1: raise ValueError(f"Bad Private Key in file {private_key}") if isinstance(certs[0], rsa.RSAPrivateKeyWithSerialization): key = josepy.jwk.JWKRSA.load(data) alg = josepy.jwa.RS256 elif isinstance(certs[0], ec.EllipticCurvePrivateKeyWithSerialization): key = josepy.jwk.JWKEC.load(data) alg = { 521: josepy.jwa.ES512, 256: josepy.jwa.ES256, 384: josepy.jwa.ES384, }[key.key._wrapped.key_size] else: raise ValueError(f"Bad Private Key in file {private_key}") return key, alg async def close(self): """Closes the client's session. The client may not be used for requests anymore after it has been closed. """ await self._session.close() async def start(self): """Starts the client's session. This method must be called after initialization and before making requests to an ACME server, as it fetches the ACME directory and registers the private key with the server. It is advised to register at least one :class:`ChallengeSolver` using :meth:`register_challenge_solver` before starting the client. """ async with self._session.get(self._directory_url, ssl=self._ssl_context) as resp: self._directory = await resp.json() if not self._challenge_solvers.keys(): logger.warning( "There is no challenge solver registered with the client. " "Certificate retrieval will likely fail.") if self._account: try: await self.account_lookup() except acme.messages.Error as e: if e.code != "accountDoesNotExist": raise await self.account_register() else: await self.account_register() async def account_register( self, email: str = None, phone: str = None, kid: str = None, hmac_key: str = None, ) -> None: """Registers an account with the CA. Also sends the given contact information and stores the account internally for subsequent requests. If the private key is already registered, then the account is only queried. It is usually not necessary to call this method as the account is registered or fetched automatically in :meth:`start`. :param email: The contact email :param phone: The contact phone number :param kid: The external account binding's key identifier :param hmac_key: The external account binding's symmetric encryption key :raises: :class:`acme.messages.Error` If the server rejects any of the contact information, the private key, or the external account binding. """ eab_credentials = (ExternalAccountBindingCredentials(kid, hmac_key) if kid and hmac_key else self.eab_credentials) try: external_account_binding = eab_credentials.create_eab( self._private_key.public_key(), self._directory) except ValueError: external_account_binding = None if self.eab_credentials.kid or self.eab_credentials.hmac_key: logger.warning( "The external account binding credentials are invalid, " "i.e. the kid or the hmac_key was not supplied. Trying without EAB." ) reg = acme.messages.Registration.from_data( email=email or self._contact.get("email"), phone=phone or self._contact.get("phone"), terms_of_service_agreed=True, external_account_binding=external_account_binding, ) resp, account_obj = await self._signed_request( reg, self._directory["newAccount"]) account_obj["kid"] = resp.headers["Location"] self._account = messages.Account.from_json(account_obj) async def account_update(self, **kwargs) -> None: """Updates the account's contact information. :param kwargs: Kwargs that are passed to :class:`acme.messages.Registration`'s constructor. May include a :class:`dict` *contact* containing new contact information or *status* set to :class:`acme.messages.STATUS_DEACTIVATED` to deactivate the account. :raises: :class:`acme.messages.Error` If the server rejects any of the contact info or the status update. """ reg = acme.messages.Registration(**kwargs) _, account_obj = await self._signed_request(reg, self._account.kid) account_obj["kid"] = self._account.kid self._account = messages.Account.from_json(account_obj) async def account_lookup(self) -> None: """Looks up an account using the stored private key. Also stores the account internally for subsequent requests. :raises: :class:`acme.messages.Error` If no account associated with the private key exists. """ reg = acme.messages.Registration.from_data( terms_of_service_agreed=True, only_return_existing=True) self._account = None # Otherwise the kid is sent instead of the JWK. Results in the request failing. resp, account_obj = await self._signed_request( reg, self._directory["newAccount"]) account_obj["kid"] = resp.headers["Location"] self._account = messages.Account.from_json(account_obj) async def order_create( self, identifiers: typing.Union[typing.List[dict], typing.List[str]] ) -> messages.Order: """Creates a new order with the given identifiers. :param identifiers: :class:`list` of identifiers that the order should contain. May either be a list of fully qualified domain names or a list of :class:`dict` containing the *type* and *name* (both :class:`str`) of each identifier. :raises: :class:`acme.messages.Error` If the server is unwilling to create an order with the requested identifiers. :returns: The new order. """ order = messages.NewOrder.from_data(identifiers=identifiers) resp, order_obj = await self._signed_request( order, self._directory["newOrder"]) order_obj["url"] = resp.headers["Location"] return messages.Order.from_json(order_obj) async def order_finalize( self, order: messages.Order, csr: "cryptography.x509.CertificateSigningRequest" ) -> messages.Order: """Finalizes the order using the given CSR. The caller needs to ensure that this method is called with :py:func:`asyncio.wait_for` and a time-out value. Otherwise it may result in an infinite loop if the CA never reports the order's status as *ready*. :param order: Order that is to be finalized. :param csr: The CSR that is submitted to apply for certificate issuance. :raises: * :class:`acme.messages.Error` If the server is unwilling to finalize the order. * :class:`aiohttp.ClientResponseError` If the order does not exist. :returns: The finalized order. """ cert_req = messages.CertificateRequest(csr=csr) while True: try: resp, order_obj = await self._signed_request( cert_req, order.finalize) break except acme.messages.Error as e: # Make sure that the order is in state READY before moving on. if e.code == "orderNotReady": await asyncio.sleep(self.FINALIZE_DELAY) else: raise e finalized = await self._poll_until( self.order_get, resp.headers["Location"], predicate=is_valid, negative_predicate=is_invalid, delay=5.0, max_tries=15, ) return finalized async def order_get(self, order_url: str) -> messages.Order: """Fetches an order given its URL. :param order_url: The order's URL. :raises: :class:`aiohttp.ClientResponseError` If the order does not exist. :return: The fetched order. """ resp, order = await self._signed_request(None, order_url) order["url"] = order_url return messages.Order.from_json(order) async def orders_get(self) -> typing.List[str]: """Fetches the account's orders list. :return: List containing the URLs of the account's orders. """ if not self._account.orders: return [] # TODO: implement chunking _, orders = await self._signed_request(None, self._account["orders"]) return orders async def authorization_get( self, authorization_url: str) -> acme.messages.Authorization: """Fetches an authorization given its URL. :param authorization_url: The authorization's URL. :raises: :class:`aiohttp.ClientResponseError` If the authorization does not exist. :return: The fetched authorization. """ resp, authorization = await self._signed_request( None, authorization_url) return acme.messages.Authorization.from_json(authorization) async def authorizations_complete(self, order: acme.messages.Order) -> None: """Completes all authorizations associated with the given order. Uses one of the registered :class:`ChallengeSolver` to complete one challenge per authorization. :param order: Order whose authorizations should be completed. :raises: :class:`CouldNotCompleteChallenge` If completion of one of the authorizations' challenges failed. """ authorizations = [ await self.authorization_get(authorization_url) for authorization_url in order.authorizations ] challenge_types = set([ ChallengeType(challenge.chall.typ) for authorization in authorizations for challenge in authorization.challenges ]) possible_types = self._challenge_solvers.keys() & challenge_types if len(possible_types) == 0: raise ValueError( f"The server offered the following challenge types but there is no solver " f"that is able to complete them: {', '.join(possible_types)}") chosen_challenge_type = possible_types.pop() solver = self._challenge_solvers[chosen_challenge_type] logger.debug( "Chosen challenge type: %s, solver: %s", chosen_challenge_type, type(solver).__name__, ) challenges_to_complete: typing.List[typing.Tuple[ acme.messages.Identifier, acme.messages.ChallengeBody]] = [] for authorization in authorizations: for challenge in authorization.challenges: if ChallengeType(challenge.chall.typ) == chosen_challenge_type: challenges_to_complete.append( (authorization.identifier, challenge)) break try: await self.challenges_complete(solver, challenges_to_complete) except Exception: await self.challenges_cleanup(solver, challenges_to_complete) raise else: await self.challenges_cleanup(solver, challenges_to_complete) # Realistically, polling for the authorizations to become valid should never fail since we have already # ensured that one challenge per authorization is valid. await asyncio.gather(*[ self._poll_until( self.authorization_get, authorization_url, predicate=is_valid, negative_predicate=is_invalid, ) for authorization_url in order.authorizations ]) async def challenges_cleanup( self, solver: ChallengeSolver, challenges: typing.List[typing.Tuple[acme.messages.Identifier, acme.messages.ChallengeBody]], ): """Cleans up after the challenges leveraging the given solver. :param solver: The challenge solver to use. :param challenges: List of identifier, challenge tuples to clean up after.""" await asyncio.gather(*[ solver.cleanup_challenge(self._private_key, identifier, challenge) for identifier, challenge in challenges ]) async def challenges_complete( self, solver: ChallengeSolver, challenges: typing.List[typing.Tuple[acme.messages.Identifier, acme.messages.ChallengeBody]], ): """Attempts to complete the challenges leveraging the given solver. :param solver: The challenge solver to use. :param challenges: List of identifier, challenge tuples to complete. :raises: :class:`CouldNotCompleteChallenge` If completion of one of the challenges failed. """ # Complete the pending challenges await asyncio.gather(*[ solver.complete_challenge(self._private_key, identifier, challenge) for (identifier, challenge) in challenges ]) # Tell the server that we are ready for challenge validation await asyncio.gather(*[ self.challenge_validate(challenge.uri) for _, challenge in challenges ]) # Poll until all challenges have become valid try: await asyncio.gather(*[ self._poll_until( self.challenge_get, challenge.uri, predicate=is_valid, negative_predicate=is_invalid, delay=5.0, max_tries=50, ) for _, challenge in challenges ]) except PollingException as e: raise CouldNotCompleteChallenge(e.obj) async def challenge_get(self, challenge_url: str) -> acme.messages.ChallengeBody: """Fetches a challenge given its URL. :param challenge_url: The challenge's URL. :raises: :class:`aiohttp.ClientResponseError` If the challenge does not exist. :return: The fetched challenge. """ _, challenge_obj = await self._signed_request(None, challenge_url) return acme.messages.ChallengeBody.from_json(challenge_obj) async def challenge_validate(self, challenge_url: str) -> None: """Initiates the given challenge's validation. :param challenge_url: The challenge's URL. :raises: :class:`aiohttp.ClientResponseError` If the challenge does not exist. """ await self._signed_request(None, challenge_url, post_as_get=False) async def certificate_get(self, order: acme.messages.Order) -> str: """Downloads the given order's certificate. :param order: The order whose certificate to download. :raises: * :class:`aiohttp.ClientResponseError` If the certificate does not exist. * :class:`ValueError` If the order has not been finalized yet, i.e. the certificate \ property is *None*. :return: The order's certificate encoded as PEM. """ if not order.certificate: raise ValueError("This order has not been finalized") _, pem = await self._signed_request(None, order.certificate) return pem async def certificate_revoke( self, certificate: "cryptography.x509.Certificate", reason: messages.RevocationReason = None, ) -> bool: """Revokes the given certificate. :param certificate: The certificate to revoke. :param reason: Optional reason for revocation. :raises: * :class:`aiohttp.ClientResponseError` If the certificate does not exist. * :class:`acme.messages.Error` If the revocation did not succeed. :return: *True* if the revocation succeeded. """ cert_rev = messages.Revocation(certificate=certificate, reason=reason) resp, _ = await self._signed_request(cert_rev, self._directory["revokeCert"]) return resp.status == 200 async def key_change(self, private_key): key, alg = self._open_key(private_key) key_change = messages.KeyChange(account=self._account["kid"], oldKey=self._private_key.public_key()) signed_key_change = messages.SignedKeyChange.from_data( key_change, key, alg, url=self._directory["keyChange"]) resp, data = await self._signed_request(signed_key_change, self._directory["keyChange"]) # data["kid"] = resp.headers["Location"] # self._account = messages.Account.from_json(data) self._private_key = key self._alg = alg def register_challenge_solver( self, challenge_solver: ChallengeSolver, ): """Registers a challenge solver with the client. The challenge solver is used to complete authorizations' challenges whose types it supports. :param challenge_solver: The challenge solver to register. :raises: :class:`ValueError` If a challenge solver is already registered that supports any of the challenge types that *challenge_solver* supports. """ for challenge_type in challenge_solver.SUPPORTED_CHALLENGES: if self._challenge_solvers.get(challenge_type): raise ValueError( f"A challenge solver for type {challenge_type} is already registered" ) else: self._challenge_solvers[challenge_type] = challenge_solver async def _poll_until( self, coro, *args, predicate=None, negative_predicate=None, delay=3.0, max_tries=5, **kwargs, ): tries = max_tries result = await coro(*args, **kwargs) while tries > 0: logger.debug("Polling %s%s, tries remaining: %d", coro.__name__, args, tries - 1) if predicate(result): break if negative_predicate(result): raise PollingException( result, f"Polling unsuccessful: {coro.__name__}{args}, {negative_predicate.__name__} became True", ) await asyncio.sleep(delay) result = await coro(*args, **kwargs) tries -= 1 else: raise PollingException( result, f"Polling unsuccessful: {coro.__name__}{args}") return result async def _get_nonce(self): async def fetch_nonce(): try: async with self._session.head(self._directory["newNonce"], ssl=self._ssl_context) as resp: logger.debug("Storing new nonce %s", resp.headers["Replay-Nonce"]) return resp.headers["Replay-Nonce"] except Exception as e: logger.exception(e) try: return self._nonces.pop() except KeyError: return await self._poll_until(fetch_nonce, predicate=lambda x: x, delay=5.0) def _wrap_in_jws(self, obj: typing.Optional[josepy.JSONDeSerializable], nonce, url, post_as_get): if post_as_get: jobj = obj.json_dumps(indent=2).encode() if obj else b"" else: jobj = b"{}" kwargs = {"nonce": acme.jose.b64decode(nonce), "url": url} if self._account is not None: kwargs["kid"] = self._account["kid"] return jws.JWS.sign(jobj, key=self._private_key, alg=self._alg, **kwargs).json_dumps(indent=2) async def _signed_request(self, obj: typing.Optional[josepy.JSONDeSerializable], url, post_as_get=True): tries = self.INVALID_NONCE_RETRIES while tries > 0: try: payload = self._wrap_in_jws(obj, await self._get_nonce(), url, post_as_get) return await self._make_request(payload, url) except acme.messages.Error as e: if e.code == "badNonce" and tries > 1: tries -= 1 continue raise e async def _make_request(self, payload, url): async with self._session.post( url, data=payload, headers={"Content-Type": "application/jose+json"}, ssl=self._ssl_context, ) as resp: if "Replay-Nonce" in resp.headers: self._nonces.add(resp.headers["Replay-Nonce"]) if 200 <= resp.status < 300 and resp.content_type == "application/json": data = await resp.json() elif resp.content_type == "application/problem+json": raise acme.messages.Error.from_json(await resp.json()) elif resp.status < 200 or resp.status >= 300: raise ClientResponseError(resp.request_info, resp.history, status=resp.status) else: data = await resp.text() logger.debug(data) return resp, data
class AioHttpClient(HttpClient): def __init__(self, *, connector=None, loop=None, cookies=None, headers=None, skip_auto_headers=None, auth=None, json_serialize=json.dumps, request_class=ClientRequest, response_class=ClientResponse, ws_response_class=ClientWebSocketResponse, version=http.HttpVersion11, cookie_jar=None, connector_owner=True, raise_for_status=False, read_timeout=sentinel, conn_timeout=None, auto_decompress=True, trust_env=False, **kwargs): """ The class packaging a class ClientSession to perform HTTP request and manager that these HTTP connection. For details of the params: http://aiohttp.readthedocs.io/en/stable/client_advanced.html#client-session """ super(AioHttpClient, self).__init__(**kwargs) self.client = ClientSession(connector=connector, loop=loop, cookies=cookies, headers=headers, skip_auto_headers=skip_auto_headers, auth=auth, json_serialize=json_serialize, request_class=request_class, response_class=response_class, ws_response_class=ws_response_class, version=version, cookie_jar=cookie_jar, connector_owner=connector_owner, raise_for_status=raise_for_status, read_timeout=read_timeout, conn_timeout=conn_timeout, auto_decompress=auto_decompress, trust_env=trust_env) def request(self, method, url, *args, **kwargs): return self.client.request(method=method, url=url, **kwargs) def get(self, url, *args, **kwargs): return self.client.get(url=url, **kwargs) def post(self, url, *args, data=None, **kwargs): return self.client.post(url=url, data=data, **kwargs) def put(self, url, *args, data=None, **kwargs): return self.client.put(url=url, data=data, **kwargs) def delete(self, url, *args, **kwargs): return self.client.delete(url=url, **kwargs) def options(self, url, *args, **kwargs): return self.client.options(url=url, **kwargs) def head(self, url, *args, **kwargs): return self.client.head(url=url, **kwargs) def patch(self, url, *args, data=None, **kwargs): return self.client.patch(url=url, data=data, **kwargs) async def close(self): await self.client.close() async def get_response(self, response): text = await response.text() return Response(url=response.url, status=response.status, charset=response.charset, content_type=response.content_type, content_length=response.content_length, reason=response.reason, headers=response.headers, text=text, selector=etree.HTML(text)) async def __aenter__(self): return self async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close()
class asyncBiliApi(object): '''B站异步接口类''' def __init__(self): headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108", "Referer": "https://www.bilibili.com/", 'Connection': 'keep-alive' } self._islogin = False self._show_name = None self._session = ClientSession(headers=headers) async def login_by_cookie(self, cookieData, checkBanned=True, strict=False) -> bool: ''' 登录并获取账户信息 cookieData dict 账户cookie checkBanned bool 检查是否被封禁 strict bool 是否严格限制cookie在.bilibili.com域名之下 ''' if strict: from yarl import URL self._session.cookie_jar.update_cookies( cookieData, URL('https://.bilibili.com')) else: self._session.cookie_jar.update_cookies(cookieData) await self.refreshInfo() if not self._islogin: return False if 'bili_jct' in cookieData: self._bili_jct = cookieData["bili_jct"] else: self._bili_jct = '' self._isBanned = None if checkBanned: code = (await self.likeCv(7793107))["code"] if code != 0 and code != 65006 and code != -404: self._isBanned = True import warnings warnings.warn(f'{self._name}:账号异常,请检查bili_jct参数是否有效或本账号是否被封禁') else: self._isBanned = False return True @property def banned(self): '''是否账号被异常封禁''' return self._isBanned @property def islogin(self): '''是否登录''' return self._islogin @property def myexp(self) -> int: '''获取登录的账户的经验''' return self._exp @property def mycoin(self) -> int: '''获取登录的账户的硬币数量''' return self._coin @property def vipType(self) -> int: '''获取登录的账户的vip类型''' return self._vip @property def name(self) -> str: '''获取用于显示的用户名''' return self._show_name @name.setter def name(self, name: str) -> None: '''设置用于显示的用户名''' self._show_name = name @property def username(self) -> str: '''获取登录的账户用户名''' return self._name @property def uid(self) -> int: '''获取登录的账户uid''' return self._uid async def refreshInfo(self) -> None: '''刷新账户信息(需要先登录)''' ret = await self.getWebNav() if ret["code"] != 0: self._islogin = False return self._islogin = True self._name = ret["data"]["uname"] self._uid = ret["data"]["mid"] self._vip = ret["data"]["vipType"] self._level = ret["data"]["level_info"]["current_level"] self._verified = ret["data"]["mobile_verified"] self._coin = ret["data"]["money"] self._exp = ret["data"]["level_info"]["current_exp"] if not self._show_name: self._show_name = self._name def refreshCookie(self) -> None: '''刷新cookie(需要先登录)''' cookies = {} keys = ("SESSDATA", "bili_jct", "DedeUserID", "LIVE_BUVID") for x in self._session.cookie_jar: if x.key in keys: cookies[x.key] = x.value self._session.cookie_jar.clear() self._session.cookie_jar.update_cookies(cookies) async def getFollowings(self, uid: int = None, pn: int = 1, ps: int = 50, order: str = 'desc', order_type: str = 'attention') -> dict: ''' 获取指定用户关注的up主 uid int 账户uid,默认为本账户,非登录账户只能获取20个*5页 pn int 页码,默认第一页 ps int 每页数量,默认50 order str 排序方式,默认desc order_type 排序类型,默认attention ''' if not uid: uid = self._uid url = f'https://api.bilibili.com/x/relation/followings?vmid={uid}&pn={pn}&ps={ps}&order={order}&order_type={order_type}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def spaceArticle( self, uid: int = None, pn: int = 1, ps: int = 30, sort: str = 'publish_time', ) -> dict: ''' 获取指定up主空间专栏投稿信息 uid int 账户uid,默认为本账户 pn int 页码,默认第一页 ps int 每页数量,默认50 sort str 排序方式,默认publish_time ''' if not uid: uid = self._uid url = f'https://api.bilibili.com/x/space/article?mid={uid}&pn={pn}&ps={ps}&sort={sort}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def spaceArcSearch(self, uid: int = None, pn: int = 1, ps: int = 100, tid: int = 0, order: str = 'pubdate', keyword: str = '') -> dict: ''' 获取指定up主空间视频投稿信息 uid int 账户uid,默认为本账户 pn int 页码,默认第一页 ps int 每页数量,默认50 tid int 分区 默认为0(所有分区) order str 排序方式,默认pubdate keyword str 关键字,默认为空 ''' if not uid: uid = self._uid url = f'https://api.bilibili.com/x/space/arc/search?mid={uid}&pn={pn}&ps={ps}&tid={tid}&order={order}&keyword={keyword}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def search(self, keyword: str = '', context: str = '', page: int = 1, tids: int = 0, order: str = '', duration: int = 0, search_type: str = 'video') -> dict: ''' 获取指定视频投稿信息 keyword str 关键字 context str 未知 page int 页码,默认第一页 tids int 分区 默认为0(所有分区) order str 排序方式,默认为空(综合排序) duration int 时长过滤,默认0(所有时长) search_type str 搜索类型,默认video(视频) ''' params = { "keyword": keyword, "context": context, "page": page, "tids": tids, "order": order, "duration": duration, "search_type": search_type, "single_column": 0, "__refresh__": "true", "tids_2": '', "_extra": '' } url = 'https://api.bilibili.com/x/web-interface/search/type' async with self._session.get(url, params=params, verify_ssl=False) as r: ret = await r.json() return ret async def followUser(self, followid: int, type: int = 1): ''' 关注或取关up主 followid int 要操作up主的uid type int 操作类型 1关注 0取关 ''' url = "https://api.vc.bilibili.com/feed/v1/feed/SetUserFollow" post_data = { "type": type, "follow": followid, "csrf_token": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def getMyGroups(self) -> dict: '''取应援团列表''' url = "https://api.vc.bilibili.com/link_group/v1/member/my_groups" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def groupSign(self, group_id: int, owner_id: int) -> dict: ''' 应援团签到 group_id int 应援团id owner_id int 应援团所有者uid ''' url = f'https://api.vc.bilibili.com/link_setting/v1/link_setting/sign_in?group_id={group_id}&owner_id={owner_id}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getRelationTags(self) -> dict: '''取关注用户分组列表''' url = "https://api.bilibili.com/x/relation/tags" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getRelationByUid(self, uid: int) -> dict: ''' 判断与某个up关系 是否关注,关注时间,是否拉黑..... uid int up主uid ''' url = f"https://api.bilibili.com/x/relation?fid={uid}" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getRelation(self, tagid: int = 0, pn: int = 1, ps: int = 50) -> dict: ''' 取关注分组内up主列表 tagid int 分组id ''' url = f"https://api.bilibili.com/x/relation/tag?tagid={tagid}&pn={pn}&ps={ps}" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getWebNav(self) -> dict: '''取导航信息''' url = "https://api.bilibili.com/x/web-interface/nav" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getReward(self) -> dict: '''取B站经验信息''' url = "https://account.bilibili.com/home/reward" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def likeCv(self, cvid: int, type=1) -> dict: ''' 点赞专栏 cvid int 专栏id type int 类型 ''' url = 'https://api.bilibili.com/x/article/like' post_data = {"id": cvid, "type": type, "csrf": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def vipPrivilegeReceive(self, type: int = 1) -> dict: ''' 领取B站大会员权益 type int 权益类型,1为B币劵,2为优惠券 ''' url = 'https://api.bilibili.com/x/vip/privilege/receive' post_data = {"type": type, "csrf": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def getUserWallet(self, platformType: int = 3) -> dict: ''' 获取账户钱包信息 platformType int 平台类型 ''' url = 'https://pay.bilibili.com/paywallet/wallet/getUserWallet' post_data = {"platformType": platformType} async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def elecPay(self, uid: int, num: int = 50) -> dict: ''' 用B币给up主充电 uid int up主uid num int 充电电池数量 ''' url = 'https://api.bilibili.com/x/ugcpay/trade/elec/pay/quick' post_data = { "elec_num": num, "up_mid": uid, "otype": 'up', "oid": uid, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def xliveFansMedal( self, page: int = 1, pageSize: int = 10, ) -> dict: ''' 获取粉丝牌 page int 直播间id pageSize int 字体颜色 ''' url = f'https://api.live.bilibili.com/fans_medal/v5/live_fans_medal/iApiMedal?page={page}&pageSize={pageSize}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveAnchorCheck(self, roomid: int) -> dict: ''' 查询直播天选时刻 roomid int 真实房间id,非短id ''' url = f'https://api.live.bilibili.com/xlive/lottery-interface/v1/Anchor/Check?roomid={roomid}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveAnchorJoin(self, id: int, gift_id: int, gift_num: int, platform: str = 'pc') -> dict: ''' 参与直播天选时刻 id int 天选时刻id gift_id int 礼物id gift_num int 礼物数量 ''' url = 'https://api.live.bilibili.com/xlive/lottery-interface/v1/Anchor/Join' post_data = { "id": id, "gift_id": gift_id, "gift_num": gift_num, "platform": platform, "csrf_token": self._bili_jct, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() #{"code":400,"data":null,"message":"余额不足","msg":"余额不足"} return ret async def xliveFeedHeartBeat(self) -> dict: '''直播心跳 feed''' url = 'https://api.live.bilibili.com/relation/v1/Feed/heartBeat' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() #{"code":0,"msg":"success","message":"success","data":{"open":1,"has_new":0,"count":0}} return ret async def xliveMsgSend( self, roomid: int, msg: str, color: int = 16777215, fontsize: int = 25, mode: int = 1, bubble: int = 0, ) -> dict: ''' 直播间发送消息 roomid int 直播间id msg str 要发送的消息 color int 字体颜色 fontsize int 字体大小 mode int 发送模式,应该是控制滚动,底部这些 bubble int 未知 ''' url = 'https://api.live.bilibili.com/msg/send' post_data = { "color": color, "fontsize": fontsize, "mode": mode, "msg": msg, "rnd": int(time.time()), "roomid": roomid, "bubble": bubble, "csrf_token": self._bili_jct, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def xliveBp2Gold(self, num: int = 5, platform: str = 'pc') -> dict: ''' B币劵购买金瓜子 num int 花费B币劵数量,目前1B币=1000金瓜子 platform str 平台 ''' #此接口抓包于网页https://link.bilibili.com/p/center/index中金瓜子购买 url = 'https://api.live.bilibili.com/xlive/revenue/v1/order/createOrder' post_data = { "platform": platform, "pay_bp": num * 1000, #兑换瓜子数量,目前1B币=1000金瓜子 "context_id": 1, #未知作用 "context_type": 11, #未知作用 "goods_id": 1, #商品id "goods_num": num, #商品数量,这里是B币数量 #"csrf_token": self._bili_jct, #"visit_id": 'acq5hn53owg0',#这两个不需要也能请求成功,csrf_token与csrf一致 "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() #返回示例{"code":1300014,"message":"b币余额不足","ttl":1,"data":null} #{"code":0,"message":"0","ttl":1,"data":{"status":2,"order_id":"2011042258413961167422787","gold":0,"bp":0}} return ret async def xliveSign(self) -> dict: '''B站直播签到''' url = "https://api.live.bilibili.com/xlive/web-ucenter/v1/sign/DoSign" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveGetRecommendList(self) -> dict: '''B站直播获取首页前10条直播''' url = f'https://api.live.bilibili.com/relation/v1/AppWeb/getRecommendList' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveGetRoomInfo(self, room_id: int) -> dict: ''' B站直播获取房间信息 room_id int 房间id ''' url = f'https://api.live.bilibili.com/xlive/web-room/v1/index/getInfoByRoom?room_id={room_id}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveGiftBagList(self) -> dict: '''B站直播获取背包礼物''' url = 'https://api.live.bilibili.com/xlive/web-room/v1/gift/bag_list' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveBagSend(self, biz_id, ruid, bag_id, gift_id, gift_num, storm_beat_id=0, price=0, platform="pc") -> dict: ''' B站直播送出背包礼物 biz_id int 房间号 ruid int up主的uid bag_id int 背包id gift_id int 背包里的礼物id gift_num int 送礼物的数量 storm_beat_id int price int 礼物价格 platform str 平台 ''' url = 'https://api.live.bilibili.com/gift/v2/live/bag_send' post_data = { "uid": self._uid, "gift_id": gift_id, "ruid": ruid, "send_ruid": 0, "gift_num": gift_num, "bag_id": bag_id, "platform": platform, "biz_code": "live", "biz_id": biz_id, #"rnd": rnd, #直播开始时间 "storm_beat_id": storm_beat_id, "price": price, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def coin(self, aid: int, num: int = 1, select_like: int = 1) -> dict: ''' 给指定av号视频投币 aid int 视频av号 num int 投币数量 select_like int 是否点赞 ''' url = "https://api.bilibili.com/x/web-interface/coin/add" post_data = { "aid": aid, "multiply": num, "select_like": select_like, "cross_domain": "true", "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def coinCv(self, cvid: int, num: int = 1, upid: int = 0, select_like: int = 1) -> dict: ''' 给指定cv号专栏投币 cvid int 专栏id num int 投币数量 upid int 专栏up主uid select_like int 是否点赞 ''' url = "https://api.bilibili.com/x/web-interface/coin/add" if upid == 0: #up主id不能为空,需要先请求一下专栏的up主 info = await self.articleViewInfo(cvid) upid = info["data"]["mid"] post_data = { "aid": cvid, "multiply": num, "select_like": select_like, "upid": upid, "avtype": 2, #专栏必为2,否则投到视频上面去了 "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def articleViewInfo(self, cvid: int) -> dict: ''' 获取专栏信息 cvid int 专栏id ''' url = f'https://api.bilibili.com/x/article/viewinfo?id={cvid}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def xliveWebHeartBeat(self, hb: str = None, pf: str = None) -> dict: ''' B站直播间心跳 hb str 请求信息(base64编码) "{周期}|{uid}|1|0" pf str 平台 "web" ''' params = {} if hb: params["hb"] = hb if pf: params["pf"] = pf url = 'https://live-trace.bilibili.com/xlive/rdata-interface/v1/heartbeat/webHeartBeat' async with self._session.get(url, params=params, verify_ssl=False) as r: ret = await r.json() return ret async def xliveGetBuvid(self) -> str: '''获得B站直播buvid参数''' #先查找cookie for x in self._session.cookie_jar: if x.key == 'LIVE_BUVID': return x.value #cookie中找不到,则请求一次直播页面 url = 'https://live.bilibili.com/3' async with self._session.head(url, verify_ssl=False) as r: cookies = r.cookies['LIVE_BUVID'] return str(cookies)[23:43] async def xliveHeartBeatX(self, id: list, device: list, ts: int, ets: int, benchmark: str, time: int, s: str) -> dict: ''' B站直播间内部心跳 id List[int] 整数数组[大分区,小分区,轮次,长位直播间] device List[str] 字符串数组[bvuid, uuid] ts int 时间戳 ets int 上次心跳时间戳timestamp benchmark str 上次心跳秘钥secret_key time int 上次心跳时间间隔 s str 加密字符串,由id, device, ets, ts, benchmark, time等参数计算出 ''' post_data = { "id": f'[{id[0]},{id[1]},{id[2]},{id[3]}]', "device": f'["{device[0]}","{device[1]}"]', "ts": ts, "ets": ets, "benchmark": benchmark, "time": time, "ua": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108', "csrf_token": self._bili_jct, "csrf": self._bili_jct, "s": s } url = 'https://live-trace.bilibili.com/xlive/data-interface/v1/x25Kn/X' async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def xliveHeartBeatE(self, id: list, device: list) -> dict: ''' B站进入直播间心跳 id List[int] 整数数组[大分区,小分区,轮次,长位直播间] device List[str] 字符串数组[bvuid, uuid] ''' post_data = { "id": f'[{id[0]},{id[1]},{id[2]},{id[3]}]', "device": f'["{device[0]}","{device[1]}"]', "ts": int(time.time() * 1000), "is_patch": 0, "heart_beat": [], #短时间多次进入直播间,is_patch为1,heart_beat传入xliveHeartBeatX所需要的所有数据 "ua": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108', "csrf_token": self._bili_jct, "csrf": self._bili_jct } url = 'https://live-trace.bilibili.com/xlive/data-interface/v1/x25Kn/E' async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def get_home_medals(self) -> dict: '''获得佩戴的勋章''' url = "https://api.live.bilibili.com/fans_medal/v1/fans_medal/get_home_medals" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def report(self, aid: int, cid: int, progres: int) -> dict: ''' B站上报视频观看进度 aid int 视频av号 cid int 视频cid号 progres int 观看秒数 ''' url = "http://api.bilibili.com/x/v2/history/report" post_data = { "aid": aid, "cid": cid, "progres": progres, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def share(self, aid) -> dict: ''' 分享指定av号视频 aid int 视频av号 ''' url = "https://api.bilibili.com/x/web-interface/share/add" post_data = {"aid": aid, "csrf": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def xliveGetStatus(self) -> dict: '''B站直播获取金银瓜子状态''' url = "https://api.live.bilibili.com/pay/v1/Exchange/getStatus" async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def silver2coin(self) -> dict: '''银瓜子兑换硬币''' url = "https://api.live.bilibili.com/pay/v1/Exchange/silver2coin" post_data = {"csrf_token": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def getRegions(self, rid=1, num=6) -> dict: ''' 获取B站分区视频信息 rid int 分区号 num int 获取视频数量 ''' url = "https://api.bilibili.com/x/web-interface/dynamic/region?ps=" + str( num) + "&rid=" + str(rid) async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def mangaClockIn(self, platform="android") -> dict: ''' 模拟B站漫画客户端签到 platform str 平台 ''' url = "https://manga.bilibili.com/twirp/activity.v1.Activity/ClockIn" post_data = {"platform": platform} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaGetPoint(self) -> dict: '''获取漫画积分''' url = f'https://manga.bilibili.com/twirp/pointshop.v1.Pointshop/GetUserPoint' async with self._session.post(url, json={}, verify_ssl=False) as r: ret = await r.json() return ret async def mangaShopExchange(self, product_id: int, point: int, product_num=1) -> dict: ''' 漫画积分商城兑换 product_id int 商品id point int 商品需要积分数量 product_num int 兑换商品数 ''' url = f'https://manga.bilibili.com/twirp/pointshop.v1.Pointshop/Exchange' post_data = { "product_id": product_id, "point": point, "product_num": product_num } async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaGetVipReward(self) -> dict: '''获取漫画大会员福利''' url = 'https://manga.bilibili.com/twirp/user.v1.User/GetVipReward' async with self._session.post(url, json={"reason_id": 1}, verify_ssl=False) as r: ret = await r.json() return ret async def mangaComrade(self, platform="web") -> dict: ''' 站友日漫画卷兑换查询 platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/activity.v1.Activity/Comrade?platform={platform}' async with self._session.post(url, json={}, verify_ssl=False) as r: ret = await r.json() return ret async def mangaPayBCoin(self, pay_amount: int, product_id=1, platform='web') -> dict: ''' B币购买漫画 pay_amount int 购买数量 product_id int 购买商品id platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/pay.v1.Pay/PayBCoin?platform={platform}' post_data = {"pay_amount": pay_amount, "product_id": product_id} async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaGetCoupons(self, not_expired=True, page_num=1, page_size=50, tab_type=1, platform="web") -> dict: ''' 获取账户中的漫读劵信息 not_expired bool page_num int 页数 page_size int 每页大小 tab_type int platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/user.v1.User/GetCoupons?platform={platform}' post_data = { "not_expired": not_expired, "page_num": page_num, "page_size": page_size, "tab_type": tab_type } async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaListFavorite(self, page_num=1, page_size=50, order=1, wait_free=0, platform='web') -> dict: ''' B站漫画追漫列表 page_num int 页数 page_size int 每页大小 order int 排序方式 wait_free int 显示等免漫画 platform str 平台 ''' url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/ListFavorite?platform={platform}' post_data = { "page_num": page_num, "page_size": page_size, "order": order, "wait_free": wait_free } async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaDetail(self, comic_id: int, device='pc', platform='web') -> dict: ''' 获取漫画信息 comic_id int 漫画id device str 设备 platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/ComicDetail?device={device}&platform={platform}' post_data = {"comic_id": comic_id} async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaGetEpisodeBuyInfo(self, ep_id: int, platform="web") -> dict: ''' 获取漫画购买信息 ep_id int 漫画章节id platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/GetEpisodeBuyInfo?platform={platform}' post_data = {"ep_id": ep_id} async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaBuyEpisode(self, ep_id: int, buy_method=1, coupon_id=0, auto_pay_gold_status=0, platform="web") -> dict: ''' 购买漫画 ep_id int 漫画章节id buy_method int 购买参数 coupon_id int 漫读劵id auto_pay_gold_status int 自动购买 platform str 平台 ''' url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/BuyEpisode?&platform={platform}' post_data = {"buy_method": buy_method, "ep_id": ep_id} if coupon_id: post_data["coupon_id"] = coupon_id if auto_pay_gold_status: post_data["auto_pay_gold_status"] = auto_pay_gold_status async with self._session.post(url, json=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def mangaAddFavorite(self, comic_id: int) -> dict: ''' 将漫画添加进追漫列表 comic_id int 漫画id ''' url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/AddFavorite' post_data = {"comic_id": comic_id} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() #{'code': 0, 'msg': '', 'data': {'first_fav_status': {'25902': True}}} return ret async def mangaAddHistory(self, comic_id: int, ep_id: int) -> dict: ''' 添加漫画观看历史 comic_id int 漫画id ep_id int 章节id ''' url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/AddHistory' post_data = {"comic_id": comic_id, "ep_id": ep_id} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() #{"code":0,"msg":"","data":{}} return ret async def activityAddTimes(self, sid: str, action_type: int) -> dict: ''' 增加B站活动的参与次数 sid str 活动的id action_type int 操作类型 ''' url = 'https://api.bilibili.com/x/activity/lottery/addtimes' post_data = { "sid": sid, "action_type": action_type, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def activityDo(self, sid: str, type: int) -> dict: ''' 参与B站活动 sid str 活动的id type int 操作类型 ''' url = 'https://api.bilibili.com/x/activity/lottery/do' post_data = {"sid": sid, "type": type, "csrf": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def activityMyTimes(self, sid: str) -> dict: ''' 获取B站活动次数 sid str 活动的id ''' url = f'https://api.bilibili.com/x/activity/lottery/mytimes?sid={sid}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def getDynamic(self, offset_dynamic_id: int = 0, type_list=268435455) -> dict: '''取B站用户动态数据''' if offset_dynamic_id: url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/dynamic_history?uid={self._uid}&offset_dynamic_id={offset_dynamic_id}&type={type_list}' else: url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/dynamic_new?uid={self._uid}&type_list={type_list}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json(content_type=None) return ret async def getDynamicDetail(self, dynamic_id: int) -> dict: ''' 获取动态内容 dynamic_id int 动态id ''' url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/get_dynamic_detail?dynamic_id={dynamic_id}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def dynamicReplyAdd(self, oid: int, message="", type=11, plat=1) -> dict: ''' 评论动态 oid int 动态id message str 评论信息 type int 评论类型,动态时原创则填11,非原创填17 plat int 平台 ''' url = "https://api.bilibili.com/x/v2/reply/add" post_data = { "oid": oid, "plat": plat, "type": type, "message": message, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def dynamicLike(self, dynamic_id: int, like: int = 1) -> dict: ''' 点赞动态 dynamic_id int 动态id like int 1为点赞,2为取消点赞 ''' url = "https://api.vc.bilibili.com/dynamic_like/v1/dynamic_like/thumb" post_data = { "uid": self._uid, "dynamic_id": dynamic_id, "up": like, "csrf_token": self._bili_jct, "csrf": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def dynamicRepost(self, dynamic_id: int, content="", extension='{"emoji_type":1}') -> dict: ''' 转发动态 dynamic_id int 动态id content str 转发评论内容 extension str_json ''' url = "https://api.vc.bilibili.com/dynamic_repost/v1/dynamic_repost/repost" post_data = { "uid": self._uid, "dynamic_id": dynamic_id, "content": content, "at_uids": '', "ctrl": '[]', "extension": extension, "csrf": self._bili_jct, "csrf_token": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() #{"code":0,"msg":"","message":"","data":{"result":0,"errmsg":"符合条件,允许发布","_gt_":0}} return ret async def dynamicRepostReply(self, rid: int, content="", type=1, repost_code=3000, From="create.comment", extension='{"emoji_type":1}') -> dict: ''' 转发动态 rid int 动态id content str 转发评论内容 type int 类型 repost_code int 转发代码 From str 转发来自 extension str_json ''' url = "https://api.vc.bilibili.com/dynamic_repost/v1/dynamic_repost/reply" post_data = { "uid": self._uid, "rid": rid, "type": type, "content": content, "extension": extension, "repost_code": repost_code, "from": From, "csrf_token": self._bili_jct } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def getSpaceDynamic(self, uid: int = 0, offset_dynamic_id: int = '') -> 'dict': ''' 取B站空间的动态列表 uid int B站用户uid ''' if uid == 0: uid = self._uid url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/space_history?host_uid={uid}&need_top=0&offset_dynamic_id={offset_dynamic_id}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def removeDynamic(self, dynamic_id: int) -> dict: ''' 删除自己的动态 dynamic_id int 动态id ''' url = 'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/rm_dynamic' post_data = {"dynamic_id": dynamic_id, "csrf_token": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def getLotteryNotice(self, dynamic_id: int) -> dict: ''' 取指定抽奖信息 dynamic_id int 抽奖动态id ''' url = f'https://api.vc.bilibili.com/lottery_svr/v1/lottery_svr/lottery_notice?dynamic_id={dynamic_id}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def juryInfo(self) -> dict: ''' 取当前账户风纪委员状态 ''' url = 'https://api.bilibili.com/x/credit/jury/jury' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def juryCaseObtain(self) -> dict: ''' 拉取一个案件用于风纪委员投票 ''' url = 'https://api.bilibili.com/x/credit/jury/caseObtain' post_data = {"csrf": self._bili_jct} async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def juryCaseInfo(self, cid: int) -> dict: ''' 获取风纪委员案件详细信息 ''' url = f'https://api.bilibili.com/x/credit/jury/caseInfo?cid={cid}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def juryVote( self, cid: int, **kwargs #非必选参数太多以可变参数列表传入 ) -> dict: ''' 风纪委员投票 cid int 案件ID 以下为可选参数,如果需要必须用参数名称的方式调用本函数 vote int 投票类型 0 未投票;1 封禁;2 否;3 弃权;4 删除 content str 理由 likes list[int] 整数数组,支持的观点 hates list[int] 整数数组,反对的观点 attr int 是否匿名 0 匿名;1 不匿名 apply_type int 是否更改原因 0 保持原来原因;1 投票给新原因 origin_reason int 原始原因 apply_reason int 新原因 1 刷屏 2 抢楼 3 发布色情低俗信息 4 发布赌博诈骗信息 5 发布违禁相关信息 6 发布垃圾广告信息 7 发布人身攻击言论 8 发布侵犯他人隐私信息 9 发布引战言论 10 发布剧透信息 11 恶意添加无关标签 12 恶意删除他人标签 13 发布色情信息 14 发布低俗信息 15 发布暴力血腥信息 16 涉及恶意投稿行为 17 发布非法网站信息 18 发布传播不实信息 19 发布怂恿教唆信息 20 恶意刷屏 21 账号违规 22 恶意抄袭 23 冒充自制原创 24 发布青少年不良内容 25 破坏网络安全 26 发布虚假误导信息 27 仿冒官方认证账号 28 发布不适宜内容 29 违反运营规则 30 恶意创建话题 31 发布违规抽奖 32 恶意冒充他人 ''' url = 'https://api.bilibili.com/x/credit/jury/vote' post_data = { "cid": cid, "csrf": self._bili_jct, **kwargs #所有可选参数 } async with self._session.post(url, data=post_data, verify_ssl=False) as r: ret = await r.json() return ret async def accInfo(self, uid: int) -> None: ''' 获取指定用户的空间个人信息 uid int 用户uid ''' url = f'https://api.bilibili.com/x/space/acc/info?mid={uid}' async with self._session.get(url, verify_ssl=False) as r: ret = await r.json() return ret async def __aenter__(self): return self async def __aexit__(self, *exc) -> None: await self.close() async def close(self) -> None: await self._session.close()
class Api: """Partially implemented sosach API.""" base_url = 'https://2ch.hk' async def init(self, app): self.session = ClientSession() async def close(self, app): await self.session.close() def get_url(self, path, query=dict()): """Create url for path and optional query.""" url = self.base_url.rstrip('/') + path if query: url = '{}?{}'.format(url, urlencode(query)) return url def catalog_url(self, board): """Create url for catalog file.""" return self.get_url('/{}/catalog.json'.format(board)) def thread_url(self, board, thread, from_): """Create url for thread.""" return self.get_url('/makaba/mobile.fcgi', dict( task='get_thread', board=board, thread=thread, num=from_ )) def file_url(self, path): """Create url for video file.""" return self.get_url(path) async def get_catalog(self, board): """Get thread list from catalog.""" url = self.catalog_url(board) logger.debug('Requesting catalog %s', url) try: async with self.session.get(url) as r: response = await r.json() return response['threads'] except ClientError as e: logger.warning('API error: url %s, %s', url, e) raise ApiError(e, url) async def get_thread(self, board, thread, from_=None): """Get messages from thread starting with from_ post.""" if from_ is None: from_ = thread url = self.thread_url(board, thread, from_) logger.debug('Requesting thread %s', url) try: async with self.session.get(url) as r: response = await r.json() return response except ClientError as e: logger.warning('API error: url %s, %s', url, e) raise ApiError(e, url) async def check_file(self, path): """Check if file exists at link.""" url = self.file_url(path) logger.debug('Checking file %s', url) try: async with self.session.head(url, allow_redirects=True) as r: if r.status != 200: return False return True except ClientError as e: logger.warning('API error: url %s, %s', url, e) raise ApiError(e, url)
async def valid_src(url: str, session: ClientSession): async with session.head(url) as resp: status = resp.status return status == 200
class DockerClient: """Simple client for querying Docker registry.""" def __init__(self, registry_host: str, repository: str) -> None: """Create a new Docker Client. Parameters ---------- registry_host: host to contact for registry. repository: name of the docker repository to query, ex: lsstsqre/sciplat-lab """ self.url = registry_host self.repository = repository self.session = ClientSession() self.headers = { "Accept": "application/vnd.docker.distribution.manifest.v2+json" } self._lookup_credentials() async def list_tags(self, authenticate: bool = True) -> List[str]: """List all the tags. Lists all the tags for the repository this client is used with. Parameters ---------- authenticate: should we try and authenticate? Used internally for retrying after successful authentication. """ url = f"https://{self.url}/v2/{self.repository}/tags/list" async with self.session.get(url, headers=self.headers) as r: logger.debug(f"List tags response: {r}") if r.status == 200: body = await r.json() logger.debug(body) return body["tags"] elif r.status == 401 and authenticate: await self._authenticate(r) return await self.list_tags(authenticate=False) else: raise DockerRegistryError(f"Unknown error listing tags {r}") async def get_image_hash(self, tag: str, authenticate: bool = True) -> str: """Get the hash of a tag. Get the associated image hash of a Docker tag. Parameters ---------- tag: the tag to inspect authenticate: should we try and authenticate? Used internally for retrying after successful authentication. Returns the hash as a string, such as "sha256:abcdef" """ url = f"https://{self.url}/v2/{self.repository}/manifests/{tag}" async with self.session.head(url, headers=self.headers) as r: logger.debug(f"Get image hash response: {r}") if r.status == 200: return r.headers["Docker-Content-Digest"] elif r.status == 401 and authenticate: await self._authenticate(r) return await self.get_image_hash(tag, authenticate=False) else: raise DockerRegistryError(f"Unknown error retrieving hash {r}") async def _authenticate(self, response: ClientResponse) -> None: """Internal method to authenticate after getting an auth challenge. Doesn't return anything but will set additional headers for future requests. Parameters ---------- response: response that contains an auth challenge. """ logger.debug(type(response)) logger.debug(f"Authenticating {response}") challenge = response.headers.get( "WWW-Authenticate", response.headers.get("Www-Authenticate")) if not challenge: raise DockerRegistryError("No authentication challenge") (challenge_type, params) = challenge.split(" ", 1) challenge_type = challenge_type.lower() if challenge_type == "basic": # Basic auth is used by the Nexus Docker Registry. if not self.username or not self.password: raise DockerRegistryError("No password for basic auth") self.headers["Authorization"] = BasicAuth( self.username, password=self.password).encode() logger.debug(f"Auth header is {self.headers}") logger.info("Authenticated with basic auth") elif challenge_type == "bearer": # Bearer is used by Docker's official registry. parts = {} for p in params.split(","): logger.debug(p) (k, v) = p.split("=") parts[k] = v.replace('"', "") url = parts["realm"] auth = None if self.username and self.password: auth = BasicAuth(self.username, password=self.password) async with self.session.get(url, auth=auth, params=parts) as r: if r.status == 200: body = await r.json() token = body["token"] self.headers["Authorization"] = f"Bearer {token}" logger.info("Authenticated with bearer token") else: raise DockerRegistryError(f"Error getting token {r}") else: raise DockerRegistryError( f"Unknown authentication challenge {challenge}") def _lookup_credentials(self) -> None: """Find credentials for the current client. Using the repository host, look for an entry in the dockerconfig that contains a username and password for authenticating. """ self.username = None self.password = None try: with open("/etc/secrets/.dockerconfigjson") as f: credstore = json.loads(f.read()) if self.url in credstore["auths"]: b64auth = credstore["auths"][self.url]["auth"] basic_auth = base64.b64decode(b64auth).decode() (self.username, self.password) = basic_auth.split(":") logger.debug(f"Found {self.url}: {self.username}") except FileNotFoundError: pass