class RestClient():
def __init__(self):
self.client = ClientSession()
def close(self):
self.client.close()
@asyncio.coroutine
def post(self, url, headers={}):
if 'content-type' not in headers:
headers['content-type'] = 'application/json'
resp = yield from self.client.post(url, headers=headers)
data = yield from resp.read()
yield from resp.release()
return data.decode('utf-8')
@asyncio.coroutine
def get(self, url, headers={}, to_json=False):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
if to_json:
data = json.loads(data.decode('utf-8'))
return data
@asyncio.coroutine
def put(self, url, headers={}):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
@asyncio.coroutine
def delete(self, url):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
@asyncio.coroutine
def patch(self, url):
resp = yield from self.client.post(url)
data = yield from resp.read()
yield from resp.release()
return json.loads(data)
async def close_self_reaction_pr(number: int, session: aiohttp.ClientSession):
# PATCH /repos/:owner/:repo/pulls/:number
pr_url = github_url(
f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/pulls/{number}")
await session.patch(pr_url, headers=HEADERS, data=b'{"state": "closed"}')
# POST /repos/:owner/:repo/issues/:number/comments
comment_url = github_url(
f"/repos/{get_config('github.repo.owner')}/{get_config('github.repo.name')}/issues/{number}/comments")
async with session.post(comment_url, headers=HEADERS, data=json.dumps({"body": "\> Reacting to your own PR"})) as resp:
content = await resp.text()
print(f"RESPONSE: {resp.status}, CONTENT: {content}")
class Bot:
'''Base class of your bot.
You should implement the ``on_message`` function.
Text may be parsed using ``parse``
'''
def __init__(self):
self.conversations = {}
self.history = []
self._client = ClientSession()
def __enter__(self):
return self
def __exit__(self, ex_type, ex_value, ex_tb):
self._client.close()
def close(self):
self._client.close()
async def parse(self, text):
async with self._client.post(
LUIS_ENDPOINT,
text.encode('utf-8'),
) as resp:
data = await resp.json()
# TODO: Extract relevant information from data
return data
@abstractmethod
async def on_message(self, conversation, text):
pass
async def _handler(self, request):
return web.Response(
b"Not implemented",
)
async def _history(self, request):
return web.Response(
'\n'.join('<p>{!r}</p>'.format(h) for h in self.history).encode('utf-8')
)
async def call(session: aiohttp.ClientSession,
params: Dict[str, Any],
*,
identity: Optional[Dict[str, str]] = None,
check: bool = False
) -> Dict[str, Any]:
"""An asyncio coroutine which makes a HTTP request to the
Ninchat Call API using the third-party aiohttp package.
If check is set, raises a ninchat.call.APIError on "error" reply
event.
"""
data = lib.request_content(params, identity=identity)
async with session.post(lib.url, data=data, headers=lib.request_headers) as r:
if r.status != HTTPStatus.OK:
r.raise_for_status()
e = await r.json()
if check:
lib.check_event(e)
return e
class TestAdminServer(AsyncTestCase):
async def setUp(self):
self.message_results = []
self.webhook_results = []
self.port = 0
self.connector = TCPConnector(limit=16, limit_per_host=4)
session_args = {"cookie_jar": DummyCookieJar(), "connector": self.connector}
self.client_session = ClientSession(
cookie_jar=DummyCookieJar(), connector=self.connector
)
async def tearDown(self):
if self.client_session:
await self.client_session.close()
self.client_session = None
async def test_debug_middleware(self):
with async_mock.patch.object(
test_module, "LOGGER", async_mock.MagicMock()
) as mock_logger:
mock_logger.isEnabledFor = async_mock.MagicMock(return_value=True)
mock_logger.debug = async_mock.MagicMock()
request = async_mock.MagicMock(
method="GET",
path_qs="/hello/world?a=1&b=2",
match_info={"match": "info"},
text=async_mock.CoroutineMock(return_value="abc123"),
)
handler = async_mock.CoroutineMock()
await test_module.debug_middleware(request, handler)
mock_logger.isEnabledFor.assert_called_once()
assert mock_logger.debug.call_count == 3
async def test_ready_middleware(self):
with async_mock.patch.object(
test_module, "LOGGER", async_mock.MagicMock()
) as mock_logger:
mock_logger.isEnabledFor = async_mock.MagicMock(return_value=True)
mock_logger.debug = async_mock.MagicMock()
mock_logger.info = async_mock.MagicMock()
mock_logger.error = async_mock.MagicMock()
request = async_mock.MagicMock(
rel_url="/", app=async_mock.MagicMock(_state={"ready": False})
)
handler = async_mock.CoroutineMock(return_value="OK")
with self.assertRaises(test_module.web.HTTPServiceUnavailable):
await test_module.ready_middleware(request, handler)
request.app._state["ready"] = True
assert await test_module.ready_middleware(request, handler) == "OK"
request.app._state["ready"] = True
handler = async_mock.CoroutineMock(
side_effect=test_module.LedgerConfigError("Bad config")
)
with self.assertRaises(test_module.LedgerConfigError):
await test_module.ready_middleware(request, handler)
request.app._state["ready"] = True
handler = async_mock.CoroutineMock(
side_effect=test_module.web.HTTPFound(location="/api/doc")
)
with self.assertRaises(test_module.web.HTTPFound):
await test_module.ready_middleware(request, handler)
request.app._state["ready"] = True
handler = async_mock.CoroutineMock(
side_effect=test_module.asyncio.CancelledError("Cancelled")
)
with self.assertRaises(test_module.asyncio.CancelledError):
await test_module.ready_middleware(request, handler)
request.app._state["ready"] = True
handler = async_mock.CoroutineMock(side_effect=KeyError("No such thing"))
with self.assertRaises(KeyError):
await test_module.ready_middleware(request, handler)
def get_admin_server(
self, settings: dict = None, context: InjectionContext = None
) -> AdminServer:
if not context:
context = InjectionContext()
if settings:
context.update_settings(settings)
# middleware is task queue xor collector: cover both over test suite
task_queue = (settings or {}).pop("task_queue", None)
plugin_registry = async_mock.MagicMock(
test_module.PluginRegistry, autospec=True
)
plugin_registry.post_process_routes = async_mock.MagicMock()
context.injector.bind_instance(test_module.PluginRegistry, plugin_registry)
collector = Collector()
context.injector.bind_instance(test_module.Collector, collector)
profile = InMemoryProfile.test_profile()
self.port = unused_port()
return AdminServer(
"0.0.0.0",
self.port,
context,
profile,
self.outbound_message_router,
self.webhook_router,
conductor_stop=async_mock.CoroutineMock(),
task_queue=TaskQueue(max_active=4) if task_queue else None,
conductor_stats=(
None if task_queue else async_mock.CoroutineMock(return_value={"a": 1})
),
)
async def outbound_message_router(self, *args):
self.message_results.append(args)
def webhook_router(self, *args):
self.webhook_results.append(args)
async def test_start_stop(self):
with self.assertRaises(AssertionError):
await self.get_admin_server().start()
settings = {"admin.admin_insecure_mode": False}
with self.assertRaises(AssertionError):
await self.get_admin_server(settings).start()
settings = {
"admin.admin_insecure_mode": True,
"admin.admin_api_key": "test-api-key",
}
with self.assertRaises(AssertionError):
await self.get_admin_server(settings).start()
settings = {
"admin.admin_insecure_mode": False,
"admin.admin_client_max_request_size": 4,
"admin.admin_api_key": "test-api-key",
}
server = self.get_admin_server(settings)
await server.start()
assert server.app._client_max_size == 4 * 1024 * 1024
with async_mock.patch.object(
server, "websocket_queues", async_mock.MagicMock()
) as mock_wsq:
mock_wsq.values = async_mock.MagicMock(
return_value=[async_mock.MagicMock(stop=async_mock.MagicMock())]
)
await server.stop()
with async_mock.patch.object(
web.TCPSite, "start", async_mock.CoroutineMock()
) as mock_start:
mock_start.side_effect = OSError("Failure to launch")
with self.assertRaises(AdminSetupError):
await self.get_admin_server(settings).start()
async def test_import_routes(self):
# this test just imports all default admin routes
# for routes with associated tests, this shouldn't make a difference in coverage
context = InjectionContext()
context.injector.bind_instance(ProtocolRegistry, ProtocolRegistry())
await DefaultContextBuilder().load_plugins(context)
server = self.get_admin_server({"admin.admin_insecure_mode": True}, context)
app = await server.make_application()
async def test_import_routes_multitenant_middleware(self):
# imports all default admin routes
context = InjectionContext()
context.injector.bind_instance(ProtocolRegistry, ProtocolRegistry())
profile = InMemoryProfile.test_profile()
context.injector.bind_instance(
test_module.BaseMultitenantManager,
test_module.BaseMultitenantManager(profile),
)
await DefaultContextBuilder().load_plugins(context)
server = self.get_admin_server(
{
"admin.admin_insecure_mode": False,
"admin.admin_api_key": "test-api-key",
},
context,
)
# cover multitenancy start code
app = await server.make_application()
app["swagger_dict"] = {}
await server.on_startup(app)
# multitenant authz
[mt_authz_middle] = [
m for m in app.middlewares if ".check_multitenant_authorization" in str(m)
]
mock_request = async_mock.MagicMock(
method="GET",
headers={"Authorization": "Bearer ..."},
path="/multitenancy/etc",
text=async_mock.CoroutineMock(return_value="abc123"),
)
with self.assertRaises(test_module.web.HTTPUnauthorized):
await mt_authz_middle(mock_request, None)
mock_request = async_mock.MagicMock(
method="GET",
headers={},
path="/protected/non-multitenancy/non-server",
text=async_mock.CoroutineMock(return_value="abc123"),
)
with self.assertRaises(test_module.web.HTTPUnauthorized):
await mt_authz_middle(mock_request, None)
mock_request = async_mock.MagicMock(
method="GET",
headers={"Authorization": "Bearer ..."},
path="/protected/non-multitenancy/non-server",
text=async_mock.CoroutineMock(return_value="abc123"),
)
mock_handler = async_mock.CoroutineMock()
await mt_authz_middle(mock_request, mock_handler)
assert mock_handler.called_once_with(mock_request)
# multitenant setup context exception paths
[setup_ctx_middle] = [m for m in app.middlewares if ".setup_context" in str(m)]
mock_request = async_mock.MagicMock(
method="GET",
headers={"Authorization": "Non-bearer ..."},
path="/protected/non-multitenancy/non-server",
text=async_mock.CoroutineMock(return_value="abc123"),
)
with self.assertRaises(test_module.web.HTTPUnauthorized):
await setup_ctx_middle(mock_request, None)
mock_request = async_mock.MagicMock(
method="GET",
headers={"Authorization": "Bearer ..."},
path="/protected/non-multitenancy/non-server",
text=async_mock.CoroutineMock(return_value="abc123"),
)
with async_mock.patch.object(
server.multitenant_manager,
"get_profile_for_token",
async_mock.CoroutineMock(),
) as mock_get_profile:
mock_get_profile.side_effect = [
test_module.MultitenantManagerError("corrupt token"),
test_module.StorageNotFoundError("out of memory"),
]
for i in range(2):
with self.assertRaises(test_module.web.HTTPUnauthorized):
await setup_ctx_middle(mock_request, None)
async def test_register_external_plugin_x(self):
context = InjectionContext()
context.injector.bind_instance(ProtocolRegistry, ProtocolRegistry())
with self.assertRaises(ValueError):
builder = DefaultContextBuilder(
settings={"external_plugins": "aries_cloudagent.nosuchmodule"}
)
await builder.load_plugins(context)
async def test_visit_insecure_mode(self):
settings = {"admin.admin_insecure_mode": True, "task_queue": True}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.post(
f"http://127.0.0.1:{self.port}/status/reset", headers={}
) as response:
assert response.status == 200
async with self.client_session.ws_connect(
f"http://127.0.0.1:{self.port}/ws"
) as ws:
result = await ws.receive_json()
assert result["topic"] == "settings"
for path in (
"",
"plugins",
"status",
"status/live",
"status/ready",
"shutdown", # mock conductor has magic-mock stop()
):
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/{path}", headers={}
) as response:
assert response.status == 200
await server.stop()
async def test_visit_secure_mode(self):
settings = {
"admin.admin_insecure_mode": False,
"admin.admin_api_key": "test-api-key",
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status", headers={"x-api-key": "wrong-key"}
) as response:
assert response.status == 401
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status",
headers={"x-api-key": "test-api-key"},
) as response:
assert response.status == 200
async with self.client_session.ws_connect(
f"http://127.0.0.1:{self.port}/ws", headers={"x-api-key": "test-api-key"}
) as ws:
result = await ws.receive_json()
assert result["topic"] == "settings"
await server.stop()
async def test_query_config(self):
settings = {
"admin.admin_insecure_mode": False,
"admin.admin_api_key": "test-api-key",
"admin.webhook_urls": ["localhost:8123/abc#secret", "localhost:8123/def"],
"multitenant.jwt_secret": "abc123",
"wallet.key": "abc123",
"wallet.rekey": "def456",
"wallet.seed": "00000000000000000000000000000000",
"wallet.storage.creds": "secret",
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/config",
headers={"x-api-key": "test-api-key"},
) as response:
config = json.loads(await response.text())["config"]
assert "admin.admin_insecure_mode" in config
assert all(
k not in config
for k in [
"admin.admin_api_key",
"multitenant.jwt_secret",
"wallet.key",
"wallet.rekey",
"wallet.seed",
"wallet.storage_creds",
]
)
assert config["admin.webhook_urls"] == [
"localhost:8123/abc",
"localhost:8123/def",
]
async def test_visit_shutting_down(self):
settings = {
"admin.admin_insecure_mode": True,
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/shutdown", headers={}
) as response:
assert response.status == 200
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status", headers={}
) as response:
assert response.status == 503
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live", headers={}
) as response:
assert response.status == 200
await server.stop()
async def test_server_health_state(self):
settings = {
"admin.admin_insecure_mode": True,
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live", headers={}
) as response:
assert response.status == 200
response_json = await response.json()
assert response_json["alive"]
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/ready", headers={}
) as response:
assert response.status == 200
response_json = await response.json()
assert response_json["ready"]
server.notify_fatal_error()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live", headers={}
) as response:
assert response.status == 503
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/ready", headers={}
) as response:
assert response.status == 503
await server.stop()
class upnp:
def __init__(self, host, session: Optional[ClientSession] = None):
self._host = host
self._mute = False
self._volume = 0
self._connected = False
if session:
self._session = session
self._managed_session = False
else:
self._session = ClientSession()
self._managed_session = True
def __enter__(self):
return self
async def _SOAPrequest(self, action, arguments, protocole):
headers = {
"SOAPAction": '"urn:schemas-upnp-org:service:{protocole}:1#{action}"'.format(
action=action, protocole=protocole
),
"content-type": "text/xml",
}
body = """<?xml version="1.0" encoding="utf-8"?>
<s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
<s:Body>
<u:{action} xmlns:u="urn:schemas-upnp-org:service:{protocole}:1">
<InstanceID>0</InstanceID>
{arguments}
</u:{action}>
</s:Body>
</s:Envelope>""".format(
action=action, arguments=arguments, protocole=protocole
)
response = None
try:
with timeout(DEFAULT_TIMEOUT):
async with self._session.post(
f"http://{self._host}:9197/upnp/control/{protocole}1",
headers=headers,
data=body,
raise_for_status=True,
) as resp:
response = await resp.content.read()
self._connected = True
except:
self._connected = False
return response
@property
def connected(self):
return self._connected
async def async_get_volume(self):
response = await self._SOAPrequest(
"GetVolume", "<Channel>Master</Channel>", "RenderingControl"
)
if response is not None:
volume_xml = response.decode("utf8")
tree = ET.fromstring(volume_xml)
for elem in tree.iter(tag="CurrentVolume"):
self._volume = elem.text
return self._volume
async def async_set_volume(self, volume):
await self._SOAPrequest(
"SetVolume",
"<Channel>Master</Channel><DesiredVolume>{}</DesiredVolume>".format(volume),
"RenderingControl",
)
async def async_get_mute(self):
response = await self._SOAPrequest(
"GetMute", "<Channel>Master</Channel>", "RenderingControl"
)
if response is not None:
# mute_xml = response.decode('utf8')
tree = ET.fromstring(response.decode("utf8"))
mute = 0
for elem in tree.iter(tag="CurrentMute"):
mute = elem.text
if int(mute) == 0:
self._mute = False
else:
self._mute = True
return self._mute
async def async_set_current_media(self, url):
""" Set media to playback and play it."""
try:
await self._SOAPrequest(
"SetAVTransportURI",
"<CurrentURI>{url}</CurrentURI><CurrentURIMetaData></CurrentURIMetaData>".format(
url=url
),
"AVTransport",
)
await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport")
except Exception:
pass
async def async_play(self):
""" Play media that was already set as current."""
try:
await self._SOAPrequest("Play", "<Speed>1</Speed>", "AVTransport")
except Exception:
pass
async def fetch(session: aiohttp.ClientSession, url: str,
body: str) -> Tuple[str, int, str]:
async with session.post(url=url, data=body) as response:
return await response.text(), response.status, response.reason
class AppStoreValidator(AppStoreValidator):
"""The asyncio version of the app store validator."""
def __init__(
self,
bundle_id: str,
sandbox: bool = False,
auto_retry_wrong_env_request: bool = False,
http_timeout: int = None,
):
super().__init__(bundle_id, sandbox, auto_retry_wrong_env_request,
http_timeout)
self._session = None
async def post_json(self, request_json: dict) -> dict:
self._change_url_by_sandbox()
if self._session is None:
# This needs to happen in a coroutine, so it's here.
self._session = ClientSession()
try:
async with self._session.post(
self.url,
json=request_json,
timeout=ClientTimeout(total=self.http_timeout),
) as resp:
return await resp.json(content_type=None)
except (ValueError, ClientError):
raise InAppPyValidationError("HTTP error")
async def validate(
self,
receipt: str,
shared_secret: str = None,
exclude_old_transactions: bool = False,
) -> dict:
""" Validates receipt against apple services.
:param receipt: receipt
:param shared_secret: optional shared secret.
:param exclude_old_transactions: optional to include only the latest renewal transaction
:return: validation result or exception.
"""
receipt_json = self._prepare_receipt(receipt, shared_secret,
exclude_old_transactions)
api_response = await self.post_json(receipt_json)
status = api_response["status"]
# Check retry case.
if self.auto_retry_wrong_env_request and status in [21007, 21008]:
# switch environment
self.sandbox = not self.sandbox
api_response = await self.post_json(receipt_json)
status = api_response["status"]
if status != api_result_ok:
error = api_result_errors.get(
status, InAppPyValidationError("Unknown API status"))
error.raw_response = api_response
raise error
return api_response
class RpcClient:
def __init__(self,
rpc_host: str,
rpc_port: int,
rpc_user: str,
rpc_password: str,
account_id: str,
secret_hash: str,
cloud_api_url: str,
target_is_cloud_api: bool,
interactive: bool = False):
self._session = ClientSession()
self.target_is_cloud_api = target_is_cloud_api
self.interactive = interactive
if target_is_cloud_api:
self._rpc_url = cloud_api_url
self._encoded_auth = base64.b64encode(
f"{account_id}:{secret_hash}".encode("utf-8")).decode("utf-8")
else:
self._rpc_url = f"http://{rpc_host}:{rpc_port}/"
self._encoded_auth = base64.b64encode(
f"{rpc_user}:{rpc_password}".encode("utf-8")).decode("utf-8")
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self.close()
async def make_request(
self,
method: RpcRequestType,
request_id: Optional[str] = None,
request_params: Union[Dict[str, Any], None] = None
) -> Optional[ClientResponse]:
if request_id is None:
request_id = str(uuid.uuid4())
json_data = {
"method": method.name.lower(),
"id": request_id,
"params": request_params
}
synchronous = True
lowercase_true = str(True).lower()
if request_params:
synchronous = \
request_params.get(rpc_constants.SYNCHRONOUS_PARAMS_KEY, lowercase_true).lower() == lowercase_true
if not synchronous and self.interactive:
asyncio.create_task(
self._session.post(self._rpc_url,
data=json.dumps(json_data),
headers={
rpc_constants.CONTENT_TYPE_HEADER_KEY:
rpc_constants.PLAIN_HEADER_TYPE,
rpc_constants.AUTHORIZATION_HEADER_KEY:
self._encoded_auth
}))
return None
return await self._session.post(
self._rpc_url,
data=json.dumps(json_data),
headers={
rpc_constants.CONTENT_TYPE_HEADER_KEY:
rpc_constants.PLAIN_HEADER_TYPE,
rpc_constants.AUTHORIZATION_HEADER_KEY: self._encoded_auth
})
async def get_server_help(self) -> ClientResponse:
return await self._session.get(
self._rpc_url,
headers={
rpc_constants.CONTENT_TYPE_HEADER_KEY:
rpc_constants.PLAIN_HEADER_TYPE,
rpc_constants.AUTHORIZATION_HEADER_KEY: self._encoded_auth
})
async def close(self) -> None:
await self._session.close()
class AsyncLCDClient:
def __init__(
self,
url: str,
chain_id: Optional[str] = None,
loop: Optional[AbstractEventLoop] = None,
_create_session: bool = True, # don't create a session (used for sync LCDClient)
):
if loop is None:
loop = get_event_loop()
self.loop = loop
if _create_session:
self.session = ClientSession(
headers={"Accept": "application/json"}, loop=self.loop
)
self.chain_id = chain_id
self.url = url
self.last_request_height = None
self.auth = AsyncAuthAPI(self)
self.bank = AsyncBankAPI(self)
self.distribution = AsyncDistributionAPI(self)
self.deployment = AsyncDeploymentAPI(self)
self.gov = AsyncGovAPI(self)
self.mint = AsyncMintAPI(self)
self.slashing = AsyncSlashingAPI(self)
self.staking = AsyncStakingAPI(self)
self.tendermint = AsyncTendermintAPI(self)
self.utils = AsyncLCDUtils(self)
async def _get(
self, endpoint: str, params: Optional[dict] = None, raw: bool = False
):
async with self.session.get(
urljoin(self.url, endpoint), params=params
) as response:
try:
result = await response.json(content_type=None)
except JSONDecodeError:
raise LCDResponseError(message=str(response.reason), response=response)
if not 200 <= response.status < 299:
raise LCDResponseError(message=result.get("error"), response=response)
self.last_request_height = result.get("height")
return result if raw else result["result"]
async def _post(
self, endpoint: str, data: Optional[dict] = None, raw: bool = False
):
async with self.session.post(
urljoin(self.url, endpoint), json=data and dict_to_data(data)
) as response:
try:
result = await response.json(content_type=None)
except JSONDecodeError:
raise LCDResponseError(message=str(response.reason), response=response)
if not 200 <= response.status < 299:
raise LCDResponseError(message=result.get("error"), response=response)
self.last_request_height = result.get("height")
return result if raw else result["result"]
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
await self.session.close()
class AuthPtc(Auth):
PTC_LOGIN_URL = 'https://sso.pokemon.com/sso/login?service=https%3A%2F%2Fsso.pokemon.com%2Fsso%2Foauth2.0%2FcallbackAuthorize'
PTC_LOGIN_OAUTH = 'https://sso.pokemon.com/sso/oauth2.0/accessToken'
PTC_LOGIN_CLIENT_SECRET = 'w8ScCUXJQc6kXKw8FiOhd8Fixzht18Dq3PEVkUCP5ZPxtgyWsbTvWHFLm2wNY0JR'
loop = get_event_loop()
def __init__(self,
username=None,
password=None,
proxy=None,
user_agent=None,
timeout=None):
Auth.__init__(self)
self._auth_provider = 'ptc'
self._session = None
self._username = username
self._password = password
self.user_agent = user_agent or 'pokemongo/0 CFNetwork/758.5.3 Darwin/15.6.0'
self.timeout = timeout or 10
if proxy and proxy.startswith('socks'):
self.socks_proxy = proxy
self.proxy = None
else:
self.socks_proxy = None
self.proxy = proxy
def activate_session(self):
if self._session and not self._session.closed:
return
if self.socks_proxy:
conn = socks_connector(self.socks_proxy, loop=self.loop)
else:
conn = TCPConnector(loop=self.loop,
verify_ssl=False,
conn_timeout=CONN_TIMEOUT)
self._session = ClientSession(connector=conn,
loop=self.loop,
headers={'User-Agent': self.user_agent})
def close_session(self):
if self._session.closed:
return
self._session.close()
async def user_login(self, username=None, password=None, retry=True):
self._username = username or self._username
self._password = password or self._password
self._login = False
if not isinstance(self._username, str) or not isinstance(
self._password, str):
raise InvalidCredentialsException(
"Username/password not correctly specified")
self.log.info('PTC User Login for: {}'.format(self._username))
self._access_token = None
self.activate_session()
try:
now = time()
async with self._session.get(self.PTC_LOGIN_URL,
timeout=self.timeout,
proxy=self.proxy) as resp:
resp.raise_for_status()
data = await resp.json(loads=json.loads)
try:
data['_eventId'] = 'submit'
data['username'] = self._username
data['password'] = self._password
except TypeError as e:
raise AuthException('Invalid initial JSON response.') from e
async with self._session.post(self.PTC_LOGIN_URL,
data=data,
timeout=self.timeout,
proxy=self.proxy,
allow_redirects=False) as resp:
resp.raise_for_status()
try:
qs = parse_qs(urlsplit(resp.headers['Location'])[3])
self._refresh_token = qs['ticket'][0]
self._access_token = resp.cookies['CASTGC'].value
except KeyError:
try:
j = await resp.json(loads=json.loads)
except jexc as e:
raise AuthException(
'Unable to decode second response.') from e
try:
if j.get('error_code'
) == 'users.login.activation_required':
raise ActivationRequiredException(
'Account email not verified.')
error = j['errors'][0]
raise AuthException(error)
except (AttributeError, KeyError, IndexError) as e:
raise AuthException(
'Unable to login or get error information.') from e
if self._access_token:
self._login = True
self._access_token_expiry = now + 7200.0
self.log.info('PTC User Login successful.')
elif self._refresh_token and retry:
return await self.get_access_token()
return self._access_token
except HttpProcessingError as e:
raise AuthConnectionException(
'Error {} during user_login: {}'.format(e.code, e.message))
except (TimeoutError, TimeoutException) as e:
raise AuthTimeoutException('user_login timeout.') from e
except (ProxyException, SocksError) as e:
raise ProxyException(
'Proxy connection error during user_login.') from e
except jexc as e:
raise AuthException('Unable to parse user_login response.') from e
except (ClientError, DisconnectedError) as e:
err = e.__cause__ or e
raise AuthConnectionException('{} during user_login.'.format(
err.__class__.__name__)) from e
except (AuthException, CancelledError):
raise
except Exception as e:
raise AuthException('{} during user_login.'.format(
e.__class__.__name__)) from e
finally:
self.close_session()
def set_refresh_token(self, refresh_token):
self.log.info('PTC Refresh Token provided by user')
self._refresh_token = refresh_token
async def get_access_token(self, force_refresh=False):
if force_refresh is False and self.check_access_token():
self.log.debug('Using cached PTC Access Token')
return self._access_token
elif self._refresh_token is None:
return await self.user_login()
else:
self.activate_session()
try:
self._login = False
self._access_token = None
if force_refresh:
self.log.info('Forced request of PTC Access Token!')
else:
self.log.info('Request PTC Access Token...')
data = {
'client_id': 'mobile-app_pokemon-go',
'redirect_uri':
'https://www.nianticlabs.com/pokemongo/error',
'client_secret': self.PTC_LOGIN_CLIENT_SECRET,
'grant_type': 'refresh_token',
'code': self._refresh_token
}
async with self._session.post(self.PTC_LOGIN_OAUTH,
data=data,
timeout=self.timeout,
proxy=self.proxy) as resp:
self._refresh_token = None
resp.raise_for_status()
qs = await resp.text()
token_data = parse_qs(qs)
try:
self._access_token = token_data['access_token'][0]
except (KeyError, IndexError):
return await self.user_login(retry=False)
if self._access_token is not None:
# set expiration to an hour less than value received because Pokemon OAuth
# login servers return an access token with an explicit expiry time of
# three hours, however, the token stops being valid after two hours.
# See issue #86
try:
self._access_token_expiry = token_data['expires'][
0] - 3600 + time()
except (KeyError, IndexError, TypeError):
self._access_token_expiry = 0
self._login = True
self.log.info('PTC Access Token successfully retrieved.')
return self._access_token
else:
self.log.info(
'Authenticating with refresh token failed, using credentials instead.'
)
return await self.user_login(retry=False)
except HttpProcessingError as e:
raise AuthConnectionException(
'Error {} while fetching access token: {}'.format(
e.code, e.message))
except (TimeoutError, TimeoutException) as e:
raise AuthTimeoutException(
'Access token request timed out.') from e
except (ProxyException, SocksError) as e:
raise ProxyException(
'Proxy connection error while fetching access token.'
) from e
except (ClientError, DisconnectedError) as e:
raise AuthConnectionException(
'{} while fetching access token.'.format(
e.__class__.__name__)) from e
except (AuthException, CancelledError):
raise
except Exception as e:
raise AuthException('{} while fetching access token.'.format(
e.__class__.__name__)) from e
finally:
self.close_session()
class Agent:
"""
An Aries CloudAgent object
This object has all the tools needed to interact with the ledger.
1. It sets-up a docker container,
2. It sets-up a webhook server to handle incoming communications
3. It has a build-in API handler to handle interaction with the ledger
"""
def __init__(
self,
identity: str,
start_port: int,
transport_protocol: str,
endpoint: str,
ledger_url: str,
local_ip: str,
wallet_name: str,
wallet_key: str,
seed: str = "random",
public_did: bool = True,
auto_response: bool = False,
):
# Construct docker container object to maintain a running container
self.docker_container = Container(
identity=identity,
endpoint=endpoint,
seed=seed,
indbound_transport_port=start_port,
outbound_transport_port=start_port + 1,
transport_protocol=transport_protocol,
wallet_name=wallet_name,
wallet_key=wallet_key,
webhook_url=
f"{transport_protocol}://{local_ip}:{start_port+2}/webhooks", #TODO:
genesis_url=f"{ledger_url}/genesis")
# Construct Api Handler object that handles all Api calls
self.api_handler = ApiHandler( # TODO: Ledger transport protocol toevoegen
api_url=local_ip,
port=start_port + 1)
# Construct a webhook server object that handles incoming messages
self.webhook_server = WebhookServer(
identity=identity,
webhook_ip=local_ip,
webhook_protocol=transport_protocol,
webhook_port=start_port + 2,
api_handler=self.api_handler
) # TODO: webhook IP is not per definitie gelijk aan ledger ip
self.identity = identity
self.start_port = start_port
self.ledger_url = ledger_url
self.local_ip = local_ip
self.seed = seed
self.public_did = public_did
self.auto_response = auto_response
# TODO: Register DID weghalen en verplaatsen naar api handler
self.client_session = ClientSession()
self.transport_protocol = transport_protocol
# TODO: random seed ja?
rand_name = str(random.randint(100_000, 999_999))
self.seed = (("my_seed_000000000000000000000000" +
rand_name)[-32:] if seed == "random" else seed)
async def initialize(self) -> None:
"""
Start a webhook server, register a DID and start a docker container process
"""
await self.webhook_server.start_process()
await self.register_did()
await self.docker_container.start_process()
# TODO: Timeout toevoegen, wanneer verkeerde key wordt gegeven, geeft hij alsog aan dat er een goeie connectie is
if self.api_handler.test_connection() is False:
return
self.admin_url = f"{self.transport_protocol}://{self.local_ip}:{self.start_port+1}"
self.endpoint = f"{self.transport_protocol}://{self.local_ip}:{self.start_port}"
log_msg(f"Admin URL is at: {self.admin_url}", color=LOG_COLOR)
log_msg(f"Endpoint URL is at: {self.endpoint}", color=LOG_COLOR)
async def connections(self,
*,
alias_query: str = None,
invitation_key_query: str = None,
my_did_query: str = None,
connection_state_query: str = None,
their_did_query: str = None,
their_role_query: str = None) -> dict:
"""
List and Query agent-to-agent connections
Function can be called with no KWARGS to list ALL conenctions
Function can also be called with KWARGS to query the list of connections
:param alias_query: Only list connections with this alias
:param invitation_key_query: Only list connections with this invitation key
:param my_did_query: Only list connections with this "my did" value
:param connection_state_query: Only list connections with this connection state
:param their_did_query: Only list connections with this "their did" value
:param their_role_query: Only list connections with this "their role" value
:return: Queried list of agent-to-agent connections with their states
"""
return self.api_handler.connections(
alias_query=alias_query,
invitation_key_query=invitation_key_query,
my_did_query=my_did_query,
connection_state_query=connection_state_query,
their_did_query=their_did_query,
their_role_query=their_did_query)
async def generate_invitation(self,
auto_accept: bool = True,
multi_use: bool = False,
display_qr: bool = False) -> tuple:
"""
Create a connection invitation
:param auto_accept: Auto accept connection handshake?
:param multi_use: Can this invite be used multiple times?
:param display_qr: Bool to indicate whether a QR code should be displayed in the terminal
:return: A tuple containing the connection id and base64 encoded invite url
"""
invitation_id, invitation = self.api_handler.create_invitation(
alias=self.identity, multi_use=multi_use, auto_accept=auto_accept)
if display_qr:
qr = QRCode(border=1)
qr.add_data(json.dumps(invitation))
log_msg(
f"Use the following JSON to accept the invite from another demo agent. Or use the QR code to connect from a mobile agent.",
color=LOG_COLOR)
log_msg(f"Invitation Data:",
json.dumps(invitation),
color=LOG_COLOR)
qr.print_ascii(invert=True)
return invitation_id, invitation
async def send_message(self, connection_id: str, message: str):
"""
Send a message to another connected agent
:param connection_id: The connection id of the connection between this agent and the connected agent
:param message: Message to send to the other agent
:return: Response of the operation
"""
connection_id = await prompt(
"Send message to <connection id> :"
) # TODO: Throw exception when invitation is invalid
message = await prompt("Message <message>:")
self.api_handler.send_message(connection_id, message)
async def create_schema(self, schema: dict = None) -> dict:
"""
Create a schema on the ACA-Py instance
:param schema: The schema to create
:return: The created schema as a dict
"""
schema = await prompt("Schema :"
) # TODO: Throw exception when schema is invalid
schema = json.loads(schema)
return self.api_handler.create_schema(schema)
async def create_credential_definition(
self,
schema_id: str = None,
schema_tag: str = None,
support_revocation: bool = False) -> str:
"""
Create a credentials definition on the Ledger
:param schema_id: The id of the schema that the credential definition needs to be created from
:param schema_tag: The tag of the schema
:param supports_revocation: Bool to indicate if the credential definition needs to support revocation or not
"""
schema_id = await prompt(
"Schema ID <id>:"
) # TODO: Throw exception when invitation is invalid
schema_tag = await prompt(
"Schema tag <tag>:"
) # TODO: Throw exception when invitation is invalid
return self.api_handler.create_credential_definition(
schema_id, schema_tag, support_revocation)
async def issue_credential(self) -> dict:
"""
Issue a credential to one of the agent. Promts the user for inputs.
:return: The result of the issue credential operation
"""
connection_id = await prompt("Connection ID <id>: ")
cred_def_id = await prompt("Connection definition ID <id>: ")
schema = json.loads(await prompt("Schema <schema>: "))
log_msg(schema)
attributes = json.loads(await prompt("Attributes <attributes>: "))
log_msg(attributes)
return self.api_handler.issue_credential(connection_id, cred_def_id,
attributes, schema)
async def get_credentials(self):
return self.api_handler.get_credentials()
async def register_did(self,
ledger_ip: str = None,
alias: str = None,
did: str = None,
verkey: str = None,
role: str = "TRUST_ANCHOR"):
"""
Function registers a DID on the ledger
:param ledger_url: The ledger_url of the ledger
:param alias: The alias to gerister on the ledger
:param did: Did to register
:param verkey: Verkey to register
:param role: role of the registered DID
:raises Exception: raises an exception when an invalid response is given by the ledger
"""
log_msg(f"Registering {self.identity} ...", color=LOG_COLOR)
data = {"alias": alias or self.identity, "role": role}
if did and verkey:
data["did"] = did
data["verkey"] = verkey
else:
data["seed"] = self.seed
async with self.client_session.post(f"{self.ledger_url}/register",
json=data) as resp:
if resp.status != 200:
raise Exception(
f"Error registering DID, response code {resp.status}")
nym_info = await resp.json()
self.did = nym_info["did"]
log_msg(f"nym_info: {nym_info}", color=LOG_COLOR)
log_msg(f"Registered DID: {self.did}", color=LOG_COLOR)
async def receive_invitation(self,
invitation,
alias=None,
auto_accept=False) -> str:
"""
Receive invitation url
:param invitation: The base64 encoded invite url str
:param alias: The alias to give to the connection as a str
:param auto_accept: Auto accept connection handshake?
:return: The connection id as a str
"""
# TODO: Throw exception when invitation is invalid
invitation = await prompt("Invite details: ")
auto_accept = await prompt("Auto accept invitation? n/y: ")
if auto_accept == "y":
auto_accept = True
else:
auto_accept = False
return self.api_handler.receive_invitation(invitation_url=invitation,
alias=self.identity,
auto_accept=True)
async def get_connection_state(self, connection_id) -> int:
"""
Get the connection state of a given connection id
:param connection_id: The connection id
:return: The state (see states dict)
"""
# TODO: Throw exception when connection_id is invalid
connection_id = await prompt("Connection state: ")
return self.api_handler.get_connection_state(connection_id)
async def terminate(self):
"""
Terminate the Agent by closing the admin API, webhook server and docker container.
:return: True if termination is complete
"""
log_msg(f"Shutting down {self.identity}", color=LOG_COLOR)
await self.client_session.close() # Close session to admin api
await self.webhook_server.terminate() # Shut down web hooks first
await self.docker_container.terminate() # now shut down the agent
return True
class Client:
# TODO: list all members here
# __slots__ = ()
def __init__(
self,
access_token: str,
app_id: str,
app_secret: str,
graph_api_version: str = GRAPH_API_VERSION,
base_graph_url: str = GRAPH_URL,
session: ClientSession = None,
loop: Optional[aio.AbstractEventLoop] = None,
):
self._access_token = access_token
self._app_id = app_id
self._app_secret = app_secret
self._graph_api_version = graph_api_version
self._base_graph_url = base_graph_url
if session is None:
self._session = ClientSession(
headers={'Content-Type': 'application/json'}, )
else:
self._session = session
if loop is None:
self._loop = self._session.loop
else:
if loop != self._session.loop:
raise RuntimeError(
"The client and session should use the same event loop.")
self._loop = loop
@property
def base_url(self):
return "{0}/v{1}".format(self._base_graph_url, self._graph_api_version)
async def get(self, endpoint: str, params: Optional[Mapping] = None):
target_url = "{0}{1}".format(
self.base_url,
endpoint,
)
if params is None:
params = {
'access_token': self._access_token,
}
async with self._session.get(target_url, params=params) as resp:
resp_data = await resp.json()
return resp_data
async def post(
self,
endpoint: str,
params: Optional[Mapping] = None,
data: Optional[Mapping] = None,
):
target_url = "{0}{1}".format(
self.base_url,
endpoint,
)
if data is None:
data = {}
if params is None:
params = {
'access_token': self._access_token,
}
async with self._session.post(target_url, params=params,
json=data) as resp:
return await resp.json()
async def send_message(
self,
messaging_type: str,
recipient: Union[str, Mapping[str, str]],
message: Mapping[str, str],
notification_type: str = 'REGULAR',
tag: str = None,
persona_id: str = None,
):
# ref: https://developers.facebook.com/docs/messenger-platform/send-messages/#messaging_types # noqa
assert messaging_type in _ALLOWED_MESSAGING_TYPE
assert notification_type in _ALLOWED_NOTIFICATION_TYPE
if isinstance(recipient, str):
recipient = {'id': recipient}
post_data = {
"messaging_type": messaging_type,
"recipient": recipient,
'message': message,
'notification_type': notification_type,
}
if persona_id is not None:
post_data['persona_id'] = persona_id
if tag is not None:
post_data['tag'] = tag
resp = await self.post('/me/messages', data=post_data)
return resp
async def send_text(
self,
recipient: Union[str, Mapping[str, str]],
text: str,
messaging_type: str = 'UPDATE',
notification_type: str = 'REGULAR',
tag: str = None,
persona_id: str = None,
):
message = {'text': text}
resp = await self.send_message(
recipient=recipient,
message=message,
messaging_type=messaging_type,
notification_type=notification_type,
tag=tag,
persona_id=persona_id,
)
return resp
async def debug_token(self):
params = {
'input_token': self._access_token,
'access_token': '{0}|{1}'.format(self._app_id, self._app_secret),
}
return await self.get('/debug_token', params=params)
def __del__(self):
self._loop.run_until_complete(self._session.close())
class Client:
def __init__(self, configuration):
self._limit_sleep_time_coefficient = configuration \
.instagram_limit_sleep_time_coefficient
self._limit_sleep_time_min = configuration \
.instagram_limit_sleep_time_min
self._success_sleep_time_coefficient = configuration \
.instagram_success_sleep_time_coefficient
self._success_sleep_time_max = configuration \
.instagram_success_sleep_time_max
self._success_sleep_time_min = configuration \
.instagram_success_sleep_time_min
self._limit_sleep_time = self._limit_sleep_time_min
self._success_sleep_time = self._success_sleep_time_max
self._username = configuration.instagram_username
self._password = configuration.instagram_password
self._referer = BASE_URL
self._session = ClientSession(
cookies={
'ig_pr': '1',
'ig_vw': '1920',
},
headers={
'User-Agent': USER_AGENT,
'X-Instagram-AJAX': '1',
'X-Requested-With': 'XMLHttpRequest',
},
)
loop = asyncio.get_event_loop()
loop.run_until_complete(self._do_login())
async def _ajax(self, url, data=None, referer=None):
"""Simulates AJAX request.
Args:
url (str): URL path. e.g.: 'query/'
data (dict, optional)
referer (str, optional): Last visited URL.
Raises:
APIError
APIFailError
APIJSONError
APILimitError
APINotAllowedError
APINotFoundError
"""
if referer is not None:
self._referer = referer
url = f'{BASE_URL}{url}'
headers = {
'Referer': self._referer,
'X-CSRFToken': self._csrf_token,
}
async with self._session.post(
url,
data=data,
headers=headers,
) as response:
if response.status == HTTPStatus.NOT_FOUND:
response.close()
await self._sleep_success()
raise APINotFoundError(f'AJAX response status code is 404 for {url}')
elif HTTPStatus.INTERNAL_SERVER_ERROR <= response.status:
response.close()
await self._sleep_success()
raise APIError(response.status)
text = await response.text()
try:
response_dict = json.loads(text)
except ValueError as err:
reason = f'AJAX request to {url} is not JSON: {err} Response ({response.status}): \"{text}\"'
if response.status == HTTPStatus.OK:
await self._sleep_success()
raise APIError(reason)
elif response.status == HTTPStatus.BAD_REQUEST:
await self._sleep_success()
raise APINotAllowedError(reason)
else:
await self._sleep_success()
raise APIError(reason)
status = response_dict.get('status')
if status == 'fail':
message = response_dict.get('message')
if isinstance(message, str) and 'temporarily blocked' in message:
await self._sleep_limit()
raise APILimitError(f'AJAX request to {url} was blocked: {response_dict}')
raise APIFailError(f'AJAX request to {url} was failed: {response_dict}')
elif status != 'ok':
raise APIError(f'AJAX request to {url} is not OK: {response_dict}')
LOGGER.debug(f'Request: {url} Response: {response_dict}')
await self._sleep_success()
return response_dict
async def _do_login(self):
"""Logins client session.
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
await self._open(BASE_URL)
self._update_csrf_token()
await self._ajax(
'accounts/login/ajax/',
data={
'username': self._username,
'password': self._password,
},
)
self._update_csrf_token()
try:
self.id = self._session.cookies['ds_user_id'].value
except KeyError as err:
reason = 'Can\'t obtain user ID from cookies.'
LOGGER.exception(reason)
raise APIError(reason) from err
async def follow(self, user):
"""
@raise APIJSONError
@raise APILimitError
@raise APINotAllowedError
@raise APINotFoundError
@raise APIError
"""
try:
await self._ajax(
'web/friendships/{}/follow/'.format(user.instagram_id),
referer=user.get_url(),
)
except APILimitError as e:
raise APILimitError(
'API limit was reached during following {}. {}'
.format(user.username, e),
)
except APIError as e:
raise APIError(
'API troubles during following {}. {}'
.format(user.username, e),
)
else:
LOGGER.debug('{} was followed'.format(user.username))
async def get_followed(self, user):
"""Fetches information about people followed by given user.
Args:
user (User): Whose subscriptions should be fetched.
Returns:
List of dicts containing following fields:
{
'id': '123',
'username': '******',
}
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
single_response_size = 50
response = await self._ajax(
'query/',
{
'q': 'ig_user({id}) {{ follows.first({count}) {{ count,'
' page_info {{ end_cursor, has_next_page }},'
' nodes {{ id, is_verified,'
' followed_by_viewer, requested_by_viewer,'
' full_name, profile_pic_url,'
' username }} }}}}'
.format(
id=user.instagram_id,
count=single_response_size,
),
'ref': 'relationships::follow_list',
},
referer=user.get_url(),
)
followed = response['follows']['nodes']
while response['follows']['page_info']['has_next_page']:
end_cursor = response['follows']['page_info']['end_cursor']
response = await self._ajax(
'query/',
{
'q': 'ig_user({id}) {{ follows.after({end_cursor},'
' {count}) {{ count, page_info {{ end_cursor,'
' has_next_page }}, nodes {{ id,'
' is_verified, followed_by_viewer,'
' requested_by_viewer, full_name,'
' profile_pic_url, username }} }}}}'
.format(
id=user.instagram_id,
end_cursor=end_cursor,
count=single_response_size,
),
'ref': 'relationships::follow_list',
},
referer=user.get_url(),
)
followed.extend(response['follows']['nodes'])
LOGGER.debug('{} followed users were fetched'.format(len(followed)))
return followed
async def _get_followers_page(self, user, cursor=None):
"""
Args:
user (User): User whose followers should be fetched
cursor: The next page to retrieve, if possible.
:param user:
:param cursor:
:return:
"""
cursor = 'first(20)' if cursor is None else \
'after({}, 20)'.format(cursor)
query = '''ig_user({user_instagram_id}) {{
followed_by.{cursor} {{
count,
page_info {{
end_cursor,
has_next_page
}},
nodes {{
id,
is_verified,
followed_by {{count}},
follows {{count}},
followed_by_viewer,
follows_viewer,
requested_by_viewer,
full_name,
profile_pic_url,
username
}}
}}
}}''' \
.format(user_instagram_id=user.instagram_id, cursor=cursor)
data = {'q': query, 'ref': 'relationships::follow_list'}
response = await self._ajax('query/', data, referer=user.get_url())
try:
followers = response['followed_by']['nodes']
page_info = response['followed_by']['page_info']
end_cursor = page_info['end_cursor']
has_next_page = page_info['has_next_page']
except (KeyError, TypeError) as e:
raise APINotAllowedError(
'Instagram have given unexpected data in '
'`_get_followers_page`. Response JSON: {response} '
'Error: {error}'.format(
response=response,
error=e,
)
)
return followers, end_cursor, has_next_page
async def get_media_by_hashtag(self, hashtag):
"""Fetches some media about specified hashtag.
Returns:
List of media IDs (strings)
Args:
hashtag (str): Hashtag to fetch
Raises:
APIError
IOError
OSError
ClientResponseError
"""
url = '{}explore/tags/{}/'.format(
BASE_URL,
urllib.parse.quote(hashtag.encode('utf-8')),
)
response = await self._session.get(url)
response = await response.read()
response = response.decode('utf-8', errors='ignore')
match = re.search(
r'<script type="text/javascript">[\w\.]+\s*=\s*([^<]+);'
'</script>',
response,
)
if match is None:
raise APIError('Can\'t find JSON in the response: {}', response)
try:
response = json.loads(match.group(1))
except ValueError as e:
raise APIError('Can\'t parse response JSON: {}'.format(e))
try:
tag = response['entry_data']['TagPage'][0]['graphql']['hashtag']
edges = tag['edge_hashtag_to_media']['edges']
media = [edge['node']['id'] for edge in edges]
except (KeyError, TypeError) as e:
raise APIError(
'Can\'t obtain media from response JSON: {}'.format(e),
)
LOGGER.debug(
'{} media about \"{}\" were fetched'.format(len(media), hashtag),
)
return media
async def get_some_followers(self, user):
"""Fetches some amount of followers of given user.
Args:
user (User): Whose followers should be fetched.
Returns:
List of dicts containing following fields:
{
'id': '123',
'username': '******',
}
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
pages_to_fetch = 3
followers = []
get_next = True
cursor = None # Eventually we will check if we have a
# cached page and use that.
LOGGER.debug('Fetching followers of {}'.format(user.username))
while get_next and pages_to_fetch > 0:
next_followers, cursor, get_next = await self._get_followers_page(
user=user,
cursor=cursor,
)
followers.extend(next_followers)
pages_to_fetch -= 1
await asyncio.sleep(5)
# TODO: Cache cursor for continuation of this, if needed.
LOGGER.debug('Fetched {} followers of {}'
.format(len(followers), user.username))
return followers
async def like(self, media):
"""
@raise APIError
@raise APIJSONError
@raise APILimitError
@raise APINotAllowedError
@raise APINotFoundError
"""
try:
await self._ajax('web/likes/{}/like/'.format(media))
except APILimitError as e:
raise APILimitError(
'API limit was reached during liking {}. {}'.format(media, e),
)
else:
LOGGER.debug('Liked {}'.format(media))
async def _open(self, url):
"""Opens given URL (HTTP GET).
Args:
url (str)
Returns:
str: Response.
"""
headers = {
'Referer': self._referer,
}
response = await self._session.get(url, headers=headers)
self._referer = url
response = await response.text()
return response
async def relogin(self):
await self._session.close()
self._session.cookies.clear()
await self._do_login()
async def _sleep_limit(self):
LOGGER.debug(
'Sleeping for {:.0f} sec because of API limits'
.format(self._limit_sleep_time),
)
await asyncio.sleep(self._limit_sleep_time)
self._limit_sleep_time *= self._limit_sleep_time_coefficient
async def _sleep_success(self):
if self._limit_sleep_time != self._limit_sleep_time_min:
self._limit_sleep_time = self._limit_sleep_time_min
self._success_sleep_time = self._success_sleep_time_max
await asyncio.sleep(self._success_sleep_time)
self._success_sleep_time = self._success_sleep_time_min + \
(self._success_sleep_time - self._success_sleep_time_min) * \
self._success_sleep_time_coefficient
async def unfollow(self, user):
"""Unfollows certain user.
Raises:
APIError
APIFailError
APIJSONError
APILimitError
APINotAllowedError
APINotFoundError
"""
try:
await self._ajax(
'web/friendships/{}/unfollow/'.format(user.instagram_id),
referer=user.get_url(),
)
except APILimitError as e:
raise APILimitError(
'API limit was reached during unfollowing {}. {}'
.format(user.username, e),
)
except APIFailError as e:
raise APIFailError(
'API troubles during unfollowing {}. {}'
.format(user.username, e),
)
except APIError as e:
raise APIError(
'API troubles during unfollowing {}. {}'
.format(user.username, e),
)
else:
LOGGER.debug('{} was unfollowed'.format(user.username))
def _update_csrf_token(self):
self._csrf_token = self._session.cookies['csrftoken'].value
LOGGER.debug('CSRF token is %s', self._csrf_token)
async def amp_go_POST(self,request,
session: aiohttp.ClientSession):
async with session.post(request['url'],
data=request['data'],headers=request['headers']) as response:
self.amp_check_status(request,response);
return await response.json()
async def _get_ublox_api_list(ublox_token: str, url: str, data: dict,
session: ClientSession) -> List[UbloxAPI]:
"""
Contacts Ublox-Api and extracts a list of UbloxApi data format (timestamps and associated raw_data)
for a specific satellite.
:param ublox_token: Token to use with UbloxApi
:param url: Url of Ublox-Api server, it could be in Italy or Sweden
:param data: asked data
:param session: Aiohttp session
:return: list of UbloxApi objects
"""
# Get logger
logger = get_logger()
try:
async with session.post(
url=url,
json=data,
headers={
"Authorization": f"Bearer {ublox_token}",
},
timeout=6,
) as resp:
# Return Info requested
return UbloxAPIList.parse_obj(await
resp.json(encoding="utf-8",
loads=orjson.loads,
content_type=None)).info
except ClientResponseError as exc:
# Token is expired
await logger.warning({
"method": exc.request_info.method,
"url": exc.request_info.url,
"token": ublox_token,
"status_code": exc.status,
"error": exc.message,
})
# Get new Token
ublox_token = await KEYCLOAK.get_ublox_token()
# Remake the request
async with session.post(
url=url,
json=data,
headers={
"Authorization": f"Bearer {ublox_token}",
},
timeout=6,
) as resp:
# Return Info requested
return UbloxAPIList.parse_obj(await
resp.json(encoding="utf-8",
loads=orjson.loads,
content_type=None)).info
except TimeoutError:
# Ublox-Api is in starvation
await logger.warning({"error": "Ublox-Api is in starvation"})
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Ublox-Api service unavailable",
)
class asyncBiliApi(object):
'''B站异步接口类'''
def __init__(self):
headers = {
"User-Agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108",
"Referer": "https://www.bilibili.com/",
'Connection': 'keep-alive'
}
self._islogin = False
self._show_name = None
self._session = ClientSession(headers=headers)
async def login_by_cookie(self,
cookieData,
checkBanned=True,
strict=False) -> bool:
'''
登录并获取账户信息
cookieData dict 账户cookie
checkBanned bool 检查是否被封禁
strict bool 是否严格限制cookie在.bilibili.com域名之下
'''
if strict:
from yarl import URL
self._session.cookie_jar.update_cookies(
cookieData, URL('https://.bilibili.com'))
else:
self._session.cookie_jar.update_cookies(cookieData)
await self.refreshInfo()
if not self._islogin:
return False
if 'bili_jct' in cookieData:
self._bili_jct = cookieData["bili_jct"]
else:
self._bili_jct = ''
self._isBanned = None
if checkBanned:
code = (await self.likeCv(7793107))["code"]
if code != 0 and code != 65006 and code != -404:
self._isBanned = True
import warnings
warnings.warn(f'{self._name}:账号异常，请检查bili_jct参数是否有效或本账号是否被封禁')
else:
self._isBanned = False
return True
@property
def banned(self):
'''是否账号被异常封禁'''
return self._isBanned
@property
def islogin(self):
'''是否登录'''
return self._islogin
@property
def myexp(self) -> int:
'''获取登录的账户的经验'''
return self._exp
@property
def mycoin(self) -> int:
'''获取登录的账户的硬币数量'''
return self._coin
@property
def vipType(self) -> int:
'''获取登录的账户的vip类型'''
return self._vip
@property
def name(self) -> str:
'''获取用于显示的用户名'''
return self._show_name
@name.setter
def name(self, name: str) -> None:
'''设置用于显示的用户名'''
self._show_name = name
@property
def username(self) -> str:
'''获取登录的账户用户名'''
return self._name
@property
def uid(self) -> int:
'''获取登录的账户uid'''
return self._uid
async def refreshInfo(self) -> None:
'''刷新账户信息(需要先登录)'''
ret = await self.getWebNav()
if ret["code"] != 0:
self._islogin = False
return
self._islogin = True
self._name = ret["data"]["uname"]
self._uid = ret["data"]["mid"]
self._vip = ret["data"]["vipType"]
self._level = ret["data"]["level_info"]["current_level"]
self._verified = ret["data"]["mobile_verified"]
self._coin = ret["data"]["money"]
self._exp = ret["data"]["level_info"]["current_exp"]
if not self._show_name:
self._show_name = self._name
def refreshCookie(self) -> None:
'''刷新cookie(需要先登录)'''
cookies = {}
keys = ("SESSDATA", "bili_jct", "DedeUserID", "LIVE_BUVID")
for x in self._session.cookie_jar:
if x.key in keys:
cookies[x.key] = x.value
self._session.cookie_jar.clear()
self._session.cookie_jar.update_cookies(cookies)
async def getFollowings(self,
uid: int = None,
pn: int = 1,
ps: int = 50,
order: str = 'desc',
order_type: str = 'attention') -> dict:
'''
获取指定用户关注的up主
uid int 账户uid，默认为本账户，非登录账户只能获取20个*5页
pn int 页码，默认第一页
ps int 每页数量，默认50
order str 排序方式，默认desc
order_type 排序类型，默认attention
'''
if not uid:
uid = self._uid
url = f'https://api.bilibili.com/x/relation/followings?vmid={uid}&pn={pn}&ps={ps}&order={order}&order_type={order_type}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def spaceArticle(
self,
uid: int = None,
pn: int = 1,
ps: int = 30,
sort: str = 'publish_time',
) -> dict:
'''
获取指定up主空间专栏投稿信息
uid int 账户uid，默认为本账户
pn int 页码，默认第一页
ps int 每页数量，默认50
sort str 排序方式，默认publish_time
'''
if not uid:
uid = self._uid
url = f'https://api.bilibili.com/x/space/article?mid={uid}&pn={pn}&ps={ps}&sort={sort}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def spaceArcSearch(self,
uid: int = None,
pn: int = 1,
ps: int = 100,
tid: int = 0,
order: str = 'pubdate',
keyword: str = '') -> dict:
'''
获取指定up主空间视频投稿信息
uid int 账户uid，默认为本账户
pn int 页码，默认第一页
ps int 每页数量，默认50
tid int 分区 默认为0(所有分区)
order str 排序方式，默认pubdate
keyword str 关键字，默认为空
'''
if not uid:
uid = self._uid
url = f'https://api.bilibili.com/x/space/arc/search?mid={uid}&pn={pn}&ps={ps}&tid={tid}&order={order}&keyword={keyword}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def search(self,
keyword: str = '',
context: str = '',
page: int = 1,
tids: int = 0,
order: str = '',
duration: int = 0,
search_type: str = 'video') -> dict:
'''
获取指定视频投稿信息
keyword str 关键字
context str 未知
page int 页码，默认第一页
tids int 分区 默认为0(所有分区)
order str 排序方式，默认为空(综合排序)
duration int 时长过滤，默认0(所有时长)
search_type str 搜索类型，默认video(视频)
'''
params = {
"keyword": keyword,
"context": context,
"page": page,
"tids": tids,
"order": order,
"duration": duration,
"search_type": search_type,
"single_column": 0,
"__refresh__": "true",
"tids_2": '',
"_extra": ''
}
url = 'https://api.bilibili.com/x/web-interface/search/type'
async with self._session.get(url, params=params,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def followUser(self, followid: int, type: int = 1):
'''
关注或取关up主
followid int 要操作up主的uid
type int 操作类型 1关注 0取关
'''
url = "https://api.vc.bilibili.com/feed/v1/feed/SetUserFollow"
post_data = {
"type": type,
"follow": followid,
"csrf_token": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def getMyGroups(self) -> dict:
'''取应援团列表'''
url = "https://api.vc.bilibili.com/link_group/v1/member/my_groups"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def groupSign(self, group_id: int, owner_id: int) -> dict:
'''
应援团签到
group_id int 应援团id
owner_id int 应援团所有者uid
'''
url = f'https://api.vc.bilibili.com/link_setting/v1/link_setting/sign_in?group_id={group_id}&owner_id={owner_id}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getRelationTags(self) -> dict:
'''取关注用户分组列表'''
url = "https://api.bilibili.com/x/relation/tags"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getRelationByUid(self, uid: int) -> dict:
'''
判断与某个up关系
是否关注，关注时间，是否拉黑.....
uid int up主uid
'''
url = f"https://api.bilibili.com/x/relation?fid={uid}"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getRelation(self,
tagid: int = 0,
pn: int = 1,
ps: int = 50) -> dict:
'''
取关注分组内up主列表
tagid int 分组id
'''
url = f"https://api.bilibili.com/x/relation/tag?tagid={tagid}&pn={pn}&ps={ps}"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getWebNav(self) -> dict:
'''取导航信息'''
url = "https://api.bilibili.com/x/web-interface/nav"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getReward(self) -> dict:
'''取B站经验信息'''
url = "https://account.bilibili.com/home/reward"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def likeCv(self, cvid: int, type=1) -> dict:
'''
点赞专栏
cvid int 专栏id
type int 类型
'''
url = 'https://api.bilibili.com/x/article/like'
post_data = {"id": cvid, "type": type, "csrf": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def vipPrivilegeReceive(self, type: int = 1) -> dict:
'''
领取B站大会员权益
type int 权益类型，1为B币劵，2为优惠券
'''
url = 'https://api.bilibili.com/x/vip/privilege/receive'
post_data = {"type": type, "csrf": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def getUserWallet(self, platformType: int = 3) -> dict:
'''
获取账户钱包信息
platformType int 平台类型
'''
url = 'https://pay.bilibili.com/paywallet/wallet/getUserWallet'
post_data = {"platformType": platformType}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def elecPay(self, uid: int, num: int = 50) -> dict:
'''
用B币给up主充电
uid int up主uid
num int 充电电池数量
'''
url = 'https://api.bilibili.com/x/ugcpay/trade/elec/pay/quick'
post_data = {
"elec_num": num,
"up_mid": uid,
"otype": 'up',
"oid": uid,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveFansMedal(
self,
page: int = 1,
pageSize: int = 10,
) -> dict:
'''
获取粉丝牌
page int 直播间id
pageSize int 字体颜色
'''
url = f'https://api.live.bilibili.com/fans_medal/v5/live_fans_medal/iApiMedal?page={page}&pageSize={pageSize}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveAnchorCheck(self, roomid: int) -> dict:
'''
查询直播天选时刻
roomid int 真实房间id，非短id
'''
url = f'https://api.live.bilibili.com/xlive/lottery-interface/v1/Anchor/Check?roomid={roomid}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveAnchorJoin(self,
id: int,
gift_id: int,
gift_num: int,
platform: str = 'pc') -> dict:
'''
参与直播天选时刻
id int 天选时刻id
gift_id int 礼物id
gift_num int 礼物数量
'''
url = 'https://api.live.bilibili.com/xlive/lottery-interface/v1/Anchor/Join'
post_data = {
"id": id,
"gift_id": gift_id,
"gift_num": gift_num,
"platform": platform,
"csrf_token": self._bili_jct,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
#{"code":400,"data":null,"message":"余额不足","msg":"余额不足"}
return ret
async def xliveFeedHeartBeat(self) -> dict:
'''直播心跳 feed'''
url = 'https://api.live.bilibili.com/relation/v1/Feed/heartBeat'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
#{"code":0,"msg":"success","message":"success","data":{"open":1,"has_new":0,"count":0}}
return ret
async def xliveMsgSend(
self,
roomid: int,
msg: str,
color: int = 16777215,
fontsize: int = 25,
mode: int = 1,
bubble: int = 0,
) -> dict:
'''
直播间发送消息
roomid int 直播间id
msg str 要发送的消息
color int 字体颜色
fontsize int 字体大小
mode int 发送模式，应该是控制滚动，底部这些
bubble int 未知
'''
url = 'https://api.live.bilibili.com/msg/send'
post_data = {
"color": color,
"fontsize": fontsize,
"mode": mode,
"msg": msg,
"rnd": int(time.time()),
"roomid": roomid,
"bubble": bubble,
"csrf_token": self._bili_jct,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveBp2Gold(self, num: int = 5, platform: str = 'pc') -> dict:
'''
B币劵购买金瓜子
num int 花费B币劵数量，目前1B币=1000金瓜子
platform str 平台
'''
#此接口抓包于网页https://link.bilibili.com/p/center/index中金瓜子购买
url = 'https://api.live.bilibili.com/xlive/revenue/v1/order/createOrder'
post_data = {
"platform": platform,
"pay_bp": num * 1000, #兑换瓜子数量，目前1B币=1000金瓜子
"context_id": 1, #未知作用
"context_type": 11, #未知作用
"goods_id": 1, #商品id
"goods_num": num, #商品数量，这里是B币数量
#"csrf_token": self._bili_jct,
#"visit_id": 'acq5hn53owg0',#这两个不需要也能请求成功，csrf_token与csrf一致
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
#返回示例{"code":1300014,"message":"b币余额不足","ttl":1,"data":null}
#{"code":0,"message":"0","ttl":1,"data":{"status":2,"order_id":"2011042258413961167422787","gold":0,"bp":0}}
return ret
async def xliveSign(self) -> dict:
'''B站直播签到'''
url = "https://api.live.bilibili.com/xlive/web-ucenter/v1/sign/DoSign"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveGetRecommendList(self) -> dict:
'''B站直播获取首页前10条直播'''
url = f'https://api.live.bilibili.com/relation/v1/AppWeb/getRecommendList'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveGetRoomInfo(self, room_id: int) -> dict:
'''
B站直播获取房间信息
room_id int 房间id
'''
url = f'https://api.live.bilibili.com/xlive/web-room/v1/index/getInfoByRoom?room_id={room_id}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveGiftBagList(self) -> dict:
'''B站直播获取背包礼物'''
url = 'https://api.live.bilibili.com/xlive/web-room/v1/gift/bag_list'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveBagSend(self,
biz_id,
ruid,
bag_id,
gift_id,
gift_num,
storm_beat_id=0,
price=0,
platform="pc") -> dict:
'''
B站直播送出背包礼物
biz_id int 房间号
ruid int up主的uid
bag_id int 背包id
gift_id int 背包里的礼物id
gift_num int 送礼物的数量
storm_beat_id int
price int 礼物价格
platform str 平台
'''
url = 'https://api.live.bilibili.com/gift/v2/live/bag_send'
post_data = {
"uid": self._uid,
"gift_id": gift_id,
"ruid": ruid,
"send_ruid": 0,
"gift_num": gift_num,
"bag_id": bag_id,
"platform": platform,
"biz_code": "live",
"biz_id": biz_id,
#"rnd": rnd, #直播开始时间
"storm_beat_id": storm_beat_id,
"price": price,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def coin(self, aid: int, num: int = 1, select_like: int = 1) -> dict:
'''
给指定av号视频投币
aid int 视频av号
num int 投币数量
select_like int 是否点赞
'''
url = "https://api.bilibili.com/x/web-interface/coin/add"
post_data = {
"aid": aid,
"multiply": num,
"select_like": select_like,
"cross_domain": "true",
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def coinCv(self,
cvid: int,
num: int = 1,
upid: int = 0,
select_like: int = 1) -> dict:
'''
给指定cv号专栏投币
cvid int 专栏id
num int 投币数量
upid int 专栏up主uid
select_like int 是否点赞
'''
url = "https://api.bilibili.com/x/web-interface/coin/add"
if upid == 0: #up主id不能为空，需要先请求一下专栏的up主
info = await self.articleViewInfo(cvid)
upid = info["data"]["mid"]
post_data = {
"aid": cvid,
"multiply": num,
"select_like": select_like,
"upid": upid,
"avtype": 2, #专栏必为2，否则投到视频上面去了
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def articleViewInfo(self, cvid: int) -> dict:
'''
获取专栏信息
cvid int 专栏id
'''
url = f'https://api.bilibili.com/x/article/viewinfo?id={cvid}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveWebHeartBeat(self, hb: str = None, pf: str = None) -> dict:
'''
B站直播间心跳
hb str 请求信息(base64编码) "{周期}|{uid}|1|0"
pf str 平台 "web"
'''
params = {}
if hb:
params["hb"] = hb
if pf:
params["pf"] = pf
url = 'https://live-trace.bilibili.com/xlive/rdata-interface/v1/heartbeat/webHeartBeat'
async with self._session.get(url, params=params,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveGetBuvid(self) -> str:
'''获得B站直播buvid参数'''
#先查找cookie
for x in self._session.cookie_jar:
if x.key == 'LIVE_BUVID':
return x.value
#cookie中找不到，则请求一次直播页面
url = 'https://live.bilibili.com/3'
async with self._session.head(url, verify_ssl=False) as r:
cookies = r.cookies['LIVE_BUVID']
return str(cookies)[23:43]
async def xliveHeartBeatX(self, id: list, device: list, ts: int, ets: int,
benchmark: str, time: int, s: str) -> dict:
'''
B站直播间内部心跳
id List[int] 整数数组[大分区,小分区,轮次,长位直播间]
device List[str] 字符串数组[bvuid, uuid]
ts int 时间戳
ets int 上次心跳时间戳timestamp
benchmark str 上次心跳秘钥secret_key
time int 上次心跳时间间隔
s str 加密字符串，由id, device, ets, ts, benchmark, time等参数计算出
'''
post_data = {
"id": f'[{id[0]},{id[1]},{id[2]},{id[3]}]',
"device": f'["{device[0]}","{device[1]}"]',
"ts": ts,
"ets": ets,
"benchmark": benchmark,
"time": time,
"ua":
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108',
"csrf_token": self._bili_jct,
"csrf": self._bili_jct,
"s": s
}
url = 'https://live-trace.bilibili.com/xlive/data-interface/v1/x25Kn/X'
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveHeartBeatE(self, id: list, device: list) -> dict:
'''
B站进入直播间心跳
id List[int] 整数数组[大分区,小分区,轮次,长位直播间]
device List[str] 字符串数组[bvuid, uuid]
'''
post_data = {
"id": f'[{id[0]},{id[1]},{id[2]},{id[3]}]',
"device": f'["{device[0]}","{device[1]}"]',
"ts": int(time.time() * 1000),
"is_patch": 0,
"heart_beat":
[], #短时间多次进入直播间，is_patch为1，heart_beat传入xliveHeartBeatX所需要的所有数据
"ua":
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/63.0.3239.108',
"csrf_token": self._bili_jct,
"csrf": self._bili_jct
}
url = 'https://live-trace.bilibili.com/xlive/data-interface/v1/x25Kn/E'
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def get_home_medals(self) -> dict:
'''获得佩戴的勋章'''
url = "https://api.live.bilibili.com/fans_medal/v1/fans_medal/get_home_medals"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def report(self, aid: int, cid: int, progres: int) -> dict:
'''
B站上报视频观看进度
aid int 视频av号
cid int 视频cid号
progres int 观看秒数
'''
url = "http://api.bilibili.com/x/v2/history/report"
post_data = {
"aid": aid,
"cid": cid,
"progres": progres,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def share(self, aid) -> dict:
'''
分享指定av号视频
aid int 视频av号
'''
url = "https://api.bilibili.com/x/web-interface/share/add"
post_data = {"aid": aid, "csrf": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def xliveGetStatus(self) -> dict:
'''B站直播获取金银瓜子状态'''
url = "https://api.live.bilibili.com/pay/v1/Exchange/getStatus"
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def silver2coin(self) -> dict:
'''银瓜子兑换硬币'''
url = "https://api.live.bilibili.com/pay/v1/Exchange/silver2coin"
post_data = {"csrf_token": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def getRegions(self, rid=1, num=6) -> dict:
'''
获取B站分区视频信息
rid int 分区号
num int 获取视频数量
'''
url = "https://api.bilibili.com/x/web-interface/dynamic/region?ps=" + str(
num) + "&rid=" + str(rid)
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaClockIn(self, platform="android") -> dict:
'''
模拟B站漫画客户端签到
platform str 平台
'''
url = "https://manga.bilibili.com/twirp/activity.v1.Activity/ClockIn"
post_data = {"platform": platform}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaGetPoint(self) -> dict:
'''获取漫画积分'''
url = f'https://manga.bilibili.com/twirp/pointshop.v1.Pointshop/GetUserPoint'
async with self._session.post(url, json={}, verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaShopExchange(self,
product_id: int,
point: int,
product_num=1) -> dict:
'''
漫画积分商城兑换
product_id int 商品id
point int 商品需要积分数量
product_num int 兑换商品数
'''
url = f'https://manga.bilibili.com/twirp/pointshop.v1.Pointshop/Exchange'
post_data = {
"product_id": product_id,
"point": point,
"product_num": product_num
}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaGetVipReward(self) -> dict:
'''获取漫画大会员福利'''
url = 'https://manga.bilibili.com/twirp/user.v1.User/GetVipReward'
async with self._session.post(url,
json={"reason_id": 1},
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaComrade(self, platform="web") -> dict:
'''
站友日漫画卷兑换查询
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/activity.v1.Activity/Comrade?platform={platform}'
async with self._session.post(url, json={}, verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaPayBCoin(self,
pay_amount: int,
product_id=1,
platform='web') -> dict:
'''
B币购买漫画
pay_amount int 购买数量
product_id int 购买商品id
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/pay.v1.Pay/PayBCoin?platform={platform}'
post_data = {"pay_amount": pay_amount, "product_id": product_id}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaGetCoupons(self,
not_expired=True,
page_num=1,
page_size=50,
tab_type=1,
platform="web") -> dict:
'''
获取账户中的漫读劵信息
not_expired bool
page_num int 页数
page_size int 每页大小
tab_type int
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/user.v1.User/GetCoupons?platform={platform}'
post_data = {
"not_expired": not_expired,
"page_num": page_num,
"page_size": page_size,
"tab_type": tab_type
}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaListFavorite(self,
page_num=1,
page_size=50,
order=1,
wait_free=0,
platform='web') -> dict:
'''
B站漫画追漫列表
page_num int 页数
page_size int 每页大小
order int 排序方式
wait_free int 显示等免漫画
platform str 平台
'''
url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/ListFavorite?platform={platform}'
post_data = {
"page_num": page_num,
"page_size": page_size,
"order": order,
"wait_free": wait_free
}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaDetail(self,
comic_id: int,
device='pc',
platform='web') -> dict:
'''
获取漫画信息
comic_id int 漫画id
device str 设备
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/ComicDetail?device={device}&platform={platform}'
post_data = {"comic_id": comic_id}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaGetEpisodeBuyInfo(self, ep_id: int, platform="web") -> dict:
'''
获取漫画购买信息
ep_id int 漫画章节id
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/GetEpisodeBuyInfo?platform={platform}'
post_data = {"ep_id": ep_id}
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaBuyEpisode(self,
ep_id: int,
buy_method=1,
coupon_id=0,
auto_pay_gold_status=0,
platform="web") -> dict:
'''
购买漫画
ep_id int 漫画章节id
buy_method int 购买参数
coupon_id int 漫读劵id
auto_pay_gold_status int 自动购买
platform str 平台
'''
url = f'https://manga.bilibili.com/twirp/comic.v1.Comic/BuyEpisode?&platform={platform}'
post_data = {"buy_method": buy_method, "ep_id": ep_id}
if coupon_id:
post_data["coupon_id"] = coupon_id
if auto_pay_gold_status:
post_data["auto_pay_gold_status"] = auto_pay_gold_status
async with self._session.post(url, json=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def mangaAddFavorite(self, comic_id: int) -> dict:
'''
将漫画添加进追漫列表
comic_id int 漫画id
'''
url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/AddFavorite'
post_data = {"comic_id": comic_id}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
#{'code': 0, 'msg': '', 'data': {'first_fav_status': {'25902': True}}}
return ret
async def mangaAddHistory(self, comic_id: int, ep_id: int) -> dict:
'''
添加漫画观看历史
comic_id int 漫画id
ep_id int 章节id
'''
url = 'https://manga.bilibili.com/twirp/bookshelf.v1.Bookshelf/AddHistory'
post_data = {"comic_id": comic_id, "ep_id": ep_id}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
#{"code":0,"msg":"","data":{}}
return ret
async def activityAddTimes(self, sid: str, action_type: int) -> dict:
'''
增加B站活动的参与次数
sid str 活动的id
action_type int 操作类型
'''
url = 'https://api.bilibili.com/x/activity/lottery/addtimes'
post_data = {
"sid": sid,
"action_type": action_type,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def activityDo(self, sid: str, type: int) -> dict:
'''
参与B站活动
sid str 活动的id
type int 操作类型
'''
url = 'https://api.bilibili.com/x/activity/lottery/do'
post_data = {"sid": sid, "type": type, "csrf": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def activityMyTimes(self, sid: str) -> dict:
'''
获取B站活动次数
sid str 活动的id
'''
url = f'https://api.bilibili.com/x/activity/lottery/mytimes?sid={sid}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def getDynamic(self,
offset_dynamic_id: int = 0,
type_list=268435455) -> dict:
'''取B站用户动态数据'''
if offset_dynamic_id:
url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/dynamic_history?uid={self._uid}&offset_dynamic_id={offset_dynamic_id}&type={type_list}'
else:
url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/dynamic_new?uid={self._uid}&type_list={type_list}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json(content_type=None)
return ret
async def getDynamicDetail(self, dynamic_id: int) -> dict:
'''
获取动态内容
dynamic_id int 动态id
'''
url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/get_dynamic_detail?dynamic_id={dynamic_id}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def dynamicReplyAdd(self,
oid: int,
message="",
type=11,
plat=1) -> dict:
'''
评论动态
oid int 动态id
message str 评论信息
type int 评论类型，动态时原创则填11，非原创填17
plat int 平台
'''
url = "https://api.bilibili.com/x/v2/reply/add"
post_data = {
"oid": oid,
"plat": plat,
"type": type,
"message": message,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def dynamicLike(self, dynamic_id: int, like: int = 1) -> dict:
'''
点赞动态
dynamic_id int 动态id
like int 1为点赞,2为取消点赞
'''
url = "https://api.vc.bilibili.com/dynamic_like/v1/dynamic_like/thumb"
post_data = {
"uid": self._uid,
"dynamic_id": dynamic_id,
"up": like,
"csrf_token": self._bili_jct,
"csrf": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def dynamicRepost(self,
dynamic_id: int,
content="",
extension='{"emoji_type":1}') -> dict:
'''
转发动态
dynamic_id int 动态id
content str 转发评论内容
extension str_json
'''
url = "https://api.vc.bilibili.com/dynamic_repost/v1/dynamic_repost/repost"
post_data = {
"uid": self._uid,
"dynamic_id": dynamic_id,
"content": content,
"at_uids": '',
"ctrl": '[]',
"extension": extension,
"csrf": self._bili_jct,
"csrf_token": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
#{"code":0,"msg":"","message":"","data":{"result":0,"errmsg":"符合条件，允许发布","_gt_":0}}
return ret
async def dynamicRepostReply(self,
rid: int,
content="",
type=1,
repost_code=3000,
From="create.comment",
extension='{"emoji_type":1}') -> dict:
'''
转发动态
rid int 动态id
content str 转发评论内容
type int 类型
repost_code int 转发代码
From str 转发来自
extension str_json
'''
url = "https://api.vc.bilibili.com/dynamic_repost/v1/dynamic_repost/reply"
post_data = {
"uid": self._uid,
"rid": rid,
"type": type,
"content": content,
"extension": extension,
"repost_code": repost_code,
"from": From,
"csrf_token": self._bili_jct
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def getSpaceDynamic(self,
uid: int = 0,
offset_dynamic_id: int = '') -> 'dict':
'''
取B站空间的动态列表
uid int B站用户uid
'''
if uid == 0:
uid = self._uid
url = f'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/space_history?host_uid={uid}&need_top=0&offset_dynamic_id={offset_dynamic_id}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def removeDynamic(self, dynamic_id: int) -> dict:
'''
删除自己的动态
dynamic_id int 动态id
'''
url = 'https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/rm_dynamic'
post_data = {"dynamic_id": dynamic_id, "csrf_token": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def getLotteryNotice(self, dynamic_id: int) -> dict:
'''
取指定抽奖信息
dynamic_id int 抽奖动态id
'''
url = f'https://api.vc.bilibili.com/lottery_svr/v1/lottery_svr/lottery_notice?dynamic_id={dynamic_id}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def juryInfo(self) -> dict:
'''
取当前账户风纪委员状态
'''
url = 'https://api.bilibili.com/x/credit/jury/jury'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def juryCaseObtain(self) -> dict:
'''
拉取一个案件用于风纪委员投票
'''
url = 'https://api.bilibili.com/x/credit/jury/caseObtain'
post_data = {"csrf": self._bili_jct}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def juryCaseInfo(self, cid: int) -> dict:
'''
获取风纪委员案件详细信息
'''
url = f'https://api.bilibili.com/x/credit/jury/caseInfo?cid={cid}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def juryVote(
self,
cid: int,
**kwargs #非必选参数太多以可变参数列表传入
) -> dict:
'''
风纪委员投票
cid int 案件ID
以下为可选参数，如果需要必须用参数名称的方式调用本函数
vote int 投票类型 0 未投票；1 封禁；2 否；3 弃权；4 删除
content str 理由
likes list[int] 整数数组，支持的观点
hates list[int] 整数数组，反对的观点
attr int 是否匿名 0 匿名；1 不匿名
apply_type int 是否更改原因 0 保持原来原因；1 投票给新原因
origin_reason int 原始原因
apply_reason int 新原因
1 刷屏
2 抢楼
3 发布色情低俗信息
4 发布赌博诈骗信息
5 发布违禁相关信息
6 发布垃圾广告信息
7 发布人身攻击言论
8 发布侵犯他人隐私信息
9 发布引战言论
10 发布剧透信息
11 恶意添加无关标签
12 恶意删除他人标签
13 发布色情信息
14 发布低俗信息
15 发布暴力血腥信息
16 涉及恶意投稿行为
17 发布非法网站信息
18 发布传播不实信息
19 发布怂恿教唆信息
20 恶意刷屏
21 账号违规
22 恶意抄袭
23 冒充自制原创
24 发布青少年不良内容
25 破坏网络安全
26 发布虚假误导信息
27 仿冒官方认证账号
28 发布不适宜内容
29 违反运营规则
30 恶意创建话题
31 发布违规抽奖
32 恶意冒充他人
'''
url = 'https://api.bilibili.com/x/credit/jury/vote'
post_data = {
"cid": cid,
"csrf": self._bili_jct,
**kwargs #所有可选参数
}
async with self._session.post(url, data=post_data,
verify_ssl=False) as r:
ret = await r.json()
return ret
async def accInfo(self, uid: int) -> None:
'''
获取指定用户的空间个人信息
uid int 用户uid
'''
url = f'https://api.bilibili.com/x/space/acc/info?mid={uid}'
async with self._session.get(url, verify_ssl=False) as r:
ret = await r.json()
return ret
async def __aenter__(self):
return self
async def __aexit__(self, *exc) -> None:
await self.close()
async def close(self) -> None:
await self._session.close()
class Client:
def __init__(self, configuration):
self._limit_sleep_time_coefficient = configuration \
.instagram_limit_sleep_time_coefficient
self._limit_sleep_time_min = configuration \
.instagram_limit_sleep_time_min
self._success_sleep_time_coefficient = configuration \
.instagram_success_sleep_time_coefficient
self._success_sleep_time_max = configuration \
.instagram_success_sleep_time_max
self._success_sleep_time_min = configuration \
.instagram_success_sleep_time_min
self._limit_sleep_time = self._limit_sleep_time_min
self._success_sleep_time = self._success_sleep_time_max
self._username = configuration.instagram_username
self._password = configuration.instagram_password
self._referer = BASE_URL
self._session = ClientSession(
cookies={
'ig_pr': '1',
'ig_vw': '1920',
},
headers={
'User-Agent': USER_AGENT,
'X-Instagram-AJAX': '1',
'X-Requested-With': 'XMLHttpRequest',
},
)
loop = asyncio.get_event_loop()
loop.run_until_complete(self._do_login())
async def _ajax(self, url, data=None, referer=None):
"""Simulates AJAX request.
Args:
url (str): URL path. e.g.: 'query/'
data (dict, optional)
referer (str, optional): Last visited URL.
Raises:
APIError
APIFailError
APIJSONError
APILimitError
APINotAllowedError
APINotFoundError
"""
if referer is not None:
self._referer = referer
url = f'{BASE_URL}{url}'
headers = {
'Referer': self._referer,
'X-CSRFToken': self._csrf_token,
}
async with self._session.post(
url,
data=data,
headers=headers,
) as response:
if response.status == HTTPStatus.NOT_FOUND:
response.close()
await self._sleep_success()
raise APINotFoundError(
f'AJAX response status code is 404 for {url}')
elif HTTPStatus.INTERNAL_SERVER_ERROR <= response.status:
response.close()
await self._sleep_success()
raise APIError(response.status)
text = await response.text()
try:
response_dict = json.loads(text)
except ValueError as err:
reason = f'AJAX request to {url} is not JSON: {err} Response ({response.status}): \"{text}\"'
if response.status == HTTPStatus.OK:
await self._sleep_success()
raise APIError(reason)
elif response.status == HTTPStatus.BAD_REQUEST:
await self._sleep_success()
raise APINotAllowedError(reason)
else:
await self._sleep_success()
raise APIError(reason)
status = response_dict.get('status')
if status == 'fail':
message = response_dict.get('message')
if isinstance(message, str) and 'temporarily blocked' in message:
await self._sleep_limit()
raise APILimitError(
f'AJAX request to {url} was blocked: {response_dict}')
raise APIFailError(
f'AJAX request to {url} was failed: {response_dict}')
elif status != 'ok':
raise APIError(f'AJAX request to {url} is not OK: {response_dict}')
LOGGER.debug(f'Request: {url} Response: {response_dict}')
await self._sleep_success()
return response_dict
async def _do_login(self):
"""Logins client session.
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
await self._open(BASE_URL)
self._update_csrf_token()
await self._ajax(
'accounts/login/ajax/',
data={
'username': self._username,
'password': self._password,
},
)
self._update_csrf_token()
try:
self.id = self._session.cookies['ds_user_id'].value
except KeyError as err:
reason = 'Can\'t obtain user ID from cookies.'
LOGGER.exception(reason)
raise APIError(reason) from err
async def follow(self, user):
"""
@raise APIJSONError
@raise APILimitError
@raise APINotAllowedError
@raise APINotFoundError
@raise APIError
"""
try:
await self._ajax(
'web/friendships/{}/follow/'.format(user.instagram_id),
referer=user.get_url(),
)
except APILimitError as e:
raise APILimitError(
'API limit was reached during following {}. {}'.format(
user.username, e), )
except APIError as e:
raise APIError(
'API troubles during following {}. {}'.format(
user.username, e), )
else:
LOGGER.debug('{} was followed'.format(user.username))
async def get_followed(self, user):
"""Fetches information about people followed by given user.
Args:
user (User): Whose subscriptions should be fetched.
Returns:
List of dicts containing following fields:
{
'id': '123',
'username': '******',
}
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
single_response_size = 50
response = await self._ajax(
'query/',
{
'q':
'ig_user({id}) {{ follows.first({count}) {{ count,'
' page_info {{ end_cursor, has_next_page }},'
' nodes {{ id, is_verified,'
' followed_by_viewer, requested_by_viewer,'
' full_name, profile_pic_url,'
' username }} }}}}'.format(
id=user.instagram_id,
count=single_response_size,
),
'ref':
'relationships::follow_list',
},
referer=user.get_url(),
)
followed = response['follows']['nodes']
while response['follows']['page_info']['has_next_page']:
end_cursor = response['follows']['page_info']['end_cursor']
response = await self._ajax(
'query/',
{
'q':
'ig_user({id}) {{ follows.after({end_cursor},'
' {count}) {{ count, page_info {{ end_cursor,'
' has_next_page }}, nodes {{ id,'
' is_verified, followed_by_viewer,'
' requested_by_viewer, full_name,'
' profile_pic_url, username }} }}}}'.format(
id=user.instagram_id,
end_cursor=end_cursor,
count=single_response_size,
),
'ref':
'relationships::follow_list',
},
referer=user.get_url(),
)
followed.extend(response['follows']['nodes'])
LOGGER.debug('{} followed users were fetched'.format(len(followed)))
return followed
async def _get_followers_page(self, user, cursor=None):
"""
Args:
user (User): User whose followers should be fetched
cursor: The next page to retrieve, if possible.
:param user:
:param cursor:
:return:
"""
cursor = 'first(20)' if cursor is None else \
'after({}, 20)'.format(cursor)
query = '''ig_user({user_instagram_id}) {{
followed_by.{cursor} {{
count,
page_info {{
end_cursor,
has_next_page
}},
nodes {{
id,
is_verified,
followed_by {{count}},
follows {{count}},
followed_by_viewer,
follows_viewer,
requested_by_viewer,
full_name,
profile_pic_url,
username
}}
}}
}}''' \
.format(user_instagram_id=user.instagram_id, cursor=cursor)
data = {'q': query, 'ref': 'relationships::follow_list'}
response = await self._ajax('query/', data, referer=user.get_url())
try:
followers = response['followed_by']['nodes']
page_info = response['followed_by']['page_info']
end_cursor = page_info['end_cursor']
has_next_page = page_info['has_next_page']
except (KeyError, TypeError) as e:
raise APINotAllowedError(
'Instagram have given unexpected data in '
'`_get_followers_page`. Response JSON: {response} '
'Error: {error}'.format(
response=response,
error=e,
))
return followers, end_cursor, has_next_page
async def get_media_by_hashtag(self, hashtag):
"""Fetches some media about specified hashtag.
Returns:
List of media IDs (strings)
Args:
hashtag (str): Hashtag to fetch
Raises:
APIError
IOError
OSError
ClientResponseError
"""
url = '{}explore/tags/{}/'.format(
BASE_URL,
urllib.parse.quote(hashtag.encode('utf-8')),
)
response = await self._session.get(url)
response = await response.read()
response = response.decode('utf-8', errors='ignore')
match = re.search(
r'<script type="text/javascript">[\w\.]+\s*=\s*([^<]+);'
'</script>',
response,
)
if match is None:
raise APIError('Can\'t find JSON in the response: {}', response)
try:
response = json.loads(match.group(1))
except ValueError as e:
raise APIError('Can\'t parse response JSON: {}'.format(e))
try:
tag = response['entry_data']['TagPage'][0]['graphql']['hashtag']
edges = tag['edge_hashtag_to_media']['edges']
media = [edge['node']['id'] for edge in edges]
except (KeyError, TypeError) as e:
raise APIError(
'Can\'t obtain media from response JSON: {}'.format(e), )
LOGGER.debug(
'{} media about \"{}\" were fetched'.format(len(media), hashtag), )
return media
async def get_some_followers(self, user):
"""Fetches some amount of followers of given user.
Args:
user (User): Whose followers should be fetched.
Returns:
List of dicts containing following fields:
{
'id': '123',
'username': '******',
}
Raises:
APIJSONError
APILimitError
APINotAllowedError
APIError
"""
pages_to_fetch = 3
followers = []
get_next = True
cursor = None # Eventually we will check if we have a
# cached page and use that.
LOGGER.debug('Fetching followers of {}'.format(user.username))
while get_next and pages_to_fetch > 0:
next_followers, cursor, get_next = await self._get_followers_page(
user=user,
cursor=cursor,
)
followers.extend(next_followers)
pages_to_fetch -= 1
await asyncio.sleep(5)
# TODO: Cache cursor for continuation of this, if needed.
LOGGER.debug('Fetched {} followers of {}'.format(
len(followers), user.username))
return followers
async def like(self, media):
"""
@raise APIError
@raise APIJSONError
@raise APILimitError
@raise APINotAllowedError
@raise APINotFoundError
"""
try:
await self._ajax('web/likes/{}/like/'.format(media))
except APILimitError as e:
raise APILimitError(
'API limit was reached during liking {}. {}'.format(media,
e), )
else:
LOGGER.debug('Liked {}'.format(media))
async def _open(self, url):
"""Opens given URL (HTTP GET).
Args:
url (str)
Returns:
str: Response.
"""
headers = {
'Referer': self._referer,
}
response = await self._session.get(url, headers=headers)
self._referer = url
response = await response.text()
return response
async def relogin(self):
await self._session.close()
self._session.cookies.clear()
await self._do_login()
async def _sleep_limit(self):
LOGGER.debug(
'Sleeping for {:.0f} sec because of API limits'.format(
self._limit_sleep_time), )
await asyncio.sleep(self._limit_sleep_time)
self._limit_sleep_time *= self._limit_sleep_time_coefficient
async def _sleep_success(self):
if self._limit_sleep_time != self._limit_sleep_time_min:
self._limit_sleep_time = self._limit_sleep_time_min
self._success_sleep_time = self._success_sleep_time_max
await asyncio.sleep(self._success_sleep_time)
self._success_sleep_time = self._success_sleep_time_min + \
(self._success_sleep_time - self._success_sleep_time_min) * \
self._success_sleep_time_coefficient
async def unfollow(self, user):
"""Unfollows certain user.
Raises:
APIError
APIFailError
APIJSONError
APILimitError
APINotAllowedError
APINotFoundError
"""
try:
await self._ajax(
'web/friendships/{}/unfollow/'.format(user.instagram_id),
referer=user.get_url(),
)
except APILimitError as e:
raise APILimitError(
'API limit was reached during unfollowing {}. {}'.format(
user.username, e), )
except APIFailError as e:
raise APIFailError(
'API troubles during unfollowing {}. {}'.format(
user.username, e), )
except APIError as e:
raise APIError(
'API troubles during unfollowing {}. {}'.format(
user.username, e), )
else:
LOGGER.debug('{} was unfollowed'.format(user.username))
def _update_csrf_token(self):
self._csrf_token = self._session.cookies['csrftoken'].value
LOGGER.debug('CSRF token is %s', self._csrf_token)
async def post(session: aiohttp.ClientSession, *args, **kwargs):
async with session.post(*args, **kwargs) as response:
return await response.text()
async def haste(session: aiohttp.ClientSession, text: str) -> str:
""" Pastes something to Hastebin, and returns the link to it. """
async with session.post(HASTEBIN_ENDPOINT, data=text) as resp:
resp_json = await resp.json()
return HASTEBIN_FMT.format(resp_json['key'])
class _Keycloak:
"""Class that handles the communication with keycloak"""
last_token_reception_time: float
"""UTC timestamp"""
last_token: str
"""Token"""
async_lock: Lock
"""Asynchronous lock """
session: ClientSession
"""Aiohttp session"""
async def setup(self):
"""
Setup keycloak, call this method only inside the startup event
"""
# timeout to obtain token
timeout = ClientTimeout(total=10)
# connection options
connector = TCPConnector(ttl_dns_cache=300, limit=1, ssl=False)
# Instantiate a session
self.session = ClientSession(
raise_for_status=True,
json_serialize=lambda x: orjson.dumps(x).decode(),
timeout=timeout,
connector=connector,
)
self.async_lock = Lock()
self.last_token = await self._get_token()
self.last_token_reception_time = time.time()
async def close(self):
"""
Close gracefully Keycloak session
"""
await self.session.close()
async def _get_token(self):
"""
Private method used to make the http request to keycloak
in order to obtain a valid token
"""
# Get Logger
logger = get_logger()
# Get settings
settings = get_keycloak_settings()
try:
async with self.session.post(
url=settings.token_request_url,
data={
"client_id": settings.client_id,
"username": settings.username_keycloak,
"password": settings.password,
"grant_type": settings.grant_type,
"client_secret": settings.client_secret,
},
) as resp:
return Token.parse_obj(
await resp.json(encoding="utf-8",
loads=orjson.loads,
content_type=None)).access_token
except ClientResponseError as exc:
await logger.error({
"method": exc.request_info.method,
"url": exc.request_info.url,
"client_id": settings.client_id,
"username": settings.username_keycloak,
"password": settings.password,
"grant_type": settings.grant_type,
"client_secret": settings.client_secret,
"status_code": exc.status,
"error": exc.message,
})
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Wrong keycloak credentials",
)
except TimeoutError:
# Keycloak is in starvation
await logger.warning({
"url": settings.token_request_url,
"error": "Keycloak is in starvation",
})
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Keycloak service not available",
)
async def get_ublox_token(self):
"""
Obtain a token from keycloak. If 150 seconds are passed since the last token was obtained,
it will obtain a fresh one and adjust it's timestamp
"""
# Obtain actual timestamp
now = time.time()
# Check if it's been at least 50 seconds since last token was obtained
if now - self.last_token_reception_time >= 150:
# Check if the lock is already acquired by a coroutine
if self.async_lock.locked():
# Await until the lock is released by the other coroutine
# and after that release it
await self.async_lock.acquire()
self.async_lock.release()
# Here we are sure that the token was updated
return self.last_token
# Only one coroutine has to update the token
async with self.async_lock:
# Update token and timestamp
self.last_token = await self._get_token()
self.last_token_reception_time = time.time()
# return the stored token
return self.last_token
class AuthTokenClient:
def __init__(
self,
connector: aiohttp.BaseConnector,
url: URL,
client_id: str,
timeout: aiohttp.ClientTimeout,
) -> None:
self._url = url
self._client_id = client_id
self._client = ClientSession(
connector=connector, connector_owner=False, timeout=timeout
)
async def close(self) -> None:
await self._client.close()
async def __aenter__(self) -> "AuthTokenClient":
return self
async def __aexit__(self, *_: Sequence[Any]) -> None:
await self.close()
async def request(self, code: AuthCode) -> _AuthToken:
payload = dict(
grant_type="authorization_code",
code_verifier=code.verifier,
code=await code.wait(),
client_id=self._client_id,
redirect_uri=str(code.callback_url),
)
async with self._client.post(self._url, json=payload) as resp:
try:
resp.raise_for_status()
except ClientResponseError as exc:
raise AuthException("failed to get an access token.") from exc
resp_payload = await resp.json()
return _AuthToken.create(
token=resp_payload["access_token"],
expires_in=resp_payload["expires_in"],
refresh_token=resp_payload["refresh_token"],
)
async def refresh(self, token: _AuthToken) -> _AuthToken:
payload = dict(
grant_type="refresh_token",
refresh_token=token.refresh_token,
client_id=self._client_id,
)
async with self._client.post(self._url, json=payload) as resp:
try:
resp.raise_for_status()
except ClientResponseError as exc:
raise AuthException("failed to get an access token.") from exc
resp_payload = await resp.json()
return _AuthToken.create(
token=resp_payload["access_token"],
expires_in=resp_payload["expires_in"],
refresh_token=token.refresh_token,
)
async def upload_dataset(session: aiohttp.ClientSession, job_id: str,
dataset: List[dict]):
path = f"/tog/tasks/?job_id={job_id}"
async with session.post(path, json=dataset) as response:
upload_response = await response.json(content_type=None)
return (upload_response, response.status)
async def refresh_token(
db: Database = Depends(get_db),
current_user: sa.UserToken = Depends(get_current_user),
aio_client: aiohttp.ClientSession = Depends(aiohttp_session),
):
try:
async with aio_client.post(
"https://bgm.tv/oauth/access_token",
data={
"grant_type": "refresh_token",
"refresh_token": current_user.refresh_token,
"client_id": config.BgmTvAutoTracker.APP_ID,
"redirect_uri": config.BgmTvAutoTracker.callback_url,
"client_secret": config.BgmTvAutoTracker.APP_SECRET,
},
) as resp:
auth_time = dateutil.parser.parse(resp.headers["Date"]).timestamp()
refresh_resp = RefreshResponse(auth_time=auth_time,
**await resp.json())
query = sa.update(
sa.UserToken,
sa.UserToken.user_id == current_user.user_id,
values={
"token_type": refresh_resp.token_type,
"scope": refresh_resp.scope or "",
"auth_time": auth_time,
"expires_in": refresh_resp.expires_in,
"access_token": refresh_resp.access_token,
"refresh_token": refresh_resp.refresh_token,
},
)
await db.execute(query)
except (
aiohttp.ServerTimeoutError,
json.decoder.JSONDecodeError,
ValidationError,
):
raise HTTPException(HTTP_502_BAD_GATEWAY,
detail="refresh user token failure")
try:
async with aio_client.get(
f"https://mirror.api.bgm.rin.cat/user/{current_user.user_id}"
) as user_info_resp:
user_info = UserInfo.parse_raw(await user_info_resp.read())
query = sa.update(
sa.UserToken,
sa.UserToken.user_id == current_user.user_id,
values={
"username": user_info.username,
"nickname": user_info.nickname,
"usergroup": user_info.usergroup.value,
},
)
await db.execute(query)
except (
aiohttp.ServerTimeoutError,
json.decoder.JSONDecodeError,
ValidationError,
) as e:
logger.exception(e)
return refresh_resp
class TestAdminServer(AsyncTestCase):
async def setUp(self):
self.message_results = []
self.webhook_results = []
self.port = 0
self.connector = TCPConnector(limit=16, limit_per_host=4)
session_args = {
"cookie_jar": DummyCookieJar(),
"connector": self.connector
}
self.client_session = ClientSession(cookie_jar=DummyCookieJar(),
connector=self.connector)
async def tearDown(self):
if self.client_session:
await self.client_session.close()
self.client_session = None
async def test_debug_middleware(self):
with async_mock.patch.object(test_module, "LOGGER",
async_mock.MagicMock()) as mock_logger:
mock_logger.isEnabledFor = async_mock.MagicMock(return_value=True)
mock_logger.debug = async_mock.MagicMock()
request = async_mock.MagicMock(
method="GET",
path_qs="/hello/world?a=1&b=2",
match_info={"match": "info"},
text=async_mock.CoroutineMock(return_value="abc123"),
)
handler = async_mock.CoroutineMock()
await test_module.debug_middleware(request, handler)
mock_logger.isEnabledFor.assert_called_once()
assert mock_logger.debug.call_count == 3
def get_admin_server(self,
settings: dict = None,
context: InjectionContext = None) -> AdminServer:
if not context:
context = InjectionContext()
if settings:
context.update_settings(settings)
# middleware is task queue xor collector: cover both over test suite
task_queue = (settings or {}).pop("task_queue", None)
plugin_registry = async_mock.MagicMock(test_module.PluginRegistry,
autospec=True)
plugin_registry.post_process_routes = async_mock.MagicMock()
context.injector.bind_instance(test_module.PluginRegistry,
plugin_registry)
collector = Collector()
context.injector.bind_instance(test_module.Collector, collector)
self.port = unused_port()
return AdminServer(
"0.0.0.0",
self.port,
context,
self.outbound_message_router,
self.webhook_router,
conductor_stop=async_mock.CoroutineMock(),
task_queue=TaskQueue(max_active=4) if task_queue else None,
conductor_stats=(None if task_queue else async_mock.CoroutineMock(
return_value=[1, 2])),
)
async def outbound_message_router(self, *args):
self.message_results.append(args)
def webhook_router(self, *args):
self.webhook_results.append(args)
async def test_start_stop(self):
with self.assertRaises(AssertionError):
await self.get_admin_server().start()
settings = {"admin.admin_insecure_mode": False}
with self.assertRaises(AssertionError):
await self.get_admin_server(settings).start()
settings = {
"admin.admin_insecure_mode": True,
"admin.admin_api_key": "test-api-key",
}
with self.assertRaises(AssertionError):
await self.get_admin_server(settings).start()
settings = {
"admin.admin_insecure_mode": False,
"admin.admin_api_key": "test-api-key",
}
server = self.get_admin_server(settings)
await server.start()
with async_mock.patch.object(server, "websocket_queues",
async_mock.MagicMock()) as mock_wsq:
mock_wsq.values = async_mock.MagicMock(return_value=[
async_mock.MagicMock(stop=async_mock.MagicMock())
])
await server.stop()
with async_mock.patch.object(web.TCPSite, "start",
async_mock.CoroutineMock()) as mock_start:
mock_start.side_effect = OSError("Failure to launch")
with self.assertRaises(AdminSetupError):
await self.get_admin_server(settings).start()
async def test_responder_send(self):
message = OutboundMessage(payload="{}")
server = self.get_admin_server()
await server.responder.send_outbound(message)
assert self.message_results == [(server.context, message)]
async def test_responder_webhook(self):
server = self.get_admin_server()
test_url = "target_url"
test_attempts = 99
server.add_webhook_target(
target_url=test_url,
topic_filter=["*"], # cover vacuous filter
max_attempts=test_attempts,
)
test_topic = "test_topic"
test_payload = {"test": "TEST"}
with async_mock.patch.object(server, "websocket_queues",
async_mock.MagicMock()) as mock_wsq:
mock_wsq.values = async_mock.MagicMock(return_value=[
async_mock.MagicMock(authenticated=True,
enqueue=async_mock.CoroutineMock())
])
await server.responder.send_webhook(test_topic, test_payload)
assert self.webhook_results == [(test_topic, test_payload,
test_url, test_attempts)]
server.remove_webhook_target(target_url=test_url)
assert test_url not in server.webhook_targets
async def test_import_routes(self):
# this test just imports all default admin routes
# for routes with associated tests, this shouldn't make a difference in coverage
context = InjectionContext()
context.injector.bind_instance(ProtocolRegistry, ProtocolRegistry())
await DefaultContextBuilder().load_plugins(context)
server = self.get_admin_server({"admin.admin_insecure_mode": True},
context)
app = await server.make_application()
async def test_register_external_plugin_x(self):
context = InjectionContext()
context.injector.bind_instance(ProtocolRegistry, ProtocolRegistry())
with self.assertRaises(ValueError):
builder = DefaultContextBuilder(
settings={"external_plugins": "aries_cloudagent.nosuchmodule"})
await builder.load_plugins(context)
async def test_visit_insecure_mode(self):
settings = {"admin.admin_insecure_mode": True, "task_queue": True}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.post(
f"http://127.0.0.1:{self.port}/status/reset",
headers={}) as response:
assert response.status == 200
async with self.client_session.ws_connect(
f"http://127.0.0.1:{self.port}/ws") as ws:
result = await ws.receive_json()
assert result["topic"] == "settings"
for path in (
"",
"plugins",
"status",
"status/live",
"status/ready",
"shutdown", # mock conductor has magic-mock stop()
):
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/{path}",
headers={}) as response:
assert response.status == 200
await server.stop()
async def test_visit_secure_mode(self):
settings = {
"admin.admin_insecure_mode": False,
"admin.admin_api_key": "test-api-key",
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status",
headers={"x-api-key": "wrong-key"}) as response:
assert response.status == 401
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status",
headers={"x-api-key": "test-api-key"},
) as response:
assert response.status == 200
async with self.client_session.ws_connect(
f"http://127.0.0.1:{self.port}/ws",
headers={"x-api-key": "test-api-key"}) as ws:
result = await ws.receive_json()
assert result["topic"] == "settings"
await server.stop()
async def test_visit_shutting_down(self):
settings = {
"admin.admin_insecure_mode": True,
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/shutdown",
headers={}) as response:
assert response.status == 200
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status",
headers={}) as response:
assert response.status == 503
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live",
headers={}) as response:
assert response.status == 200
await server.stop()
async def test_server_health_state(self):
settings = {
"admin.admin_insecure_mode": True,
}
server = self.get_admin_server(settings)
await server.start()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live",
headers={}) as response:
assert response.status == 200
response_json = await response.json()
assert response_json["alive"]
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/ready",
headers={}) as response:
assert response.status == 200
response_json = await response.json()
assert response_json["ready"]
server.notify_fatal_error()
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/live",
headers={}) as response:
assert response.status == 200
response_json = await response.json()
assert not response_json["alive"]
async with self.client_session.get(
f"http://127.0.0.1:{self.port}/status/ready",
headers={}) as response:
assert response.status == 200
response_json = await response.json()
assert not response_json["ready"]
await server.stop()
async def oauth_callback(
code: str = None,
db: Database = Depends(get_db),
redis: PickleRedis = Depends(get_redis),
aio_client: aiohttp.ClientSession = Depends(aiohttp_session),
):
redirect_response = RedirectResponse("./bgm.tv_auth")
if code is None:
return redirect_response
try:
async with aio_client.post(
"https://bgm.tv/oauth/access_token",
data={
"code": code,
"client_id": config.BgmTvAutoTracker.APP_ID,
"grant_type": "authorization_code",
"redirect_uri": CALLBACK_URL,
"client_secret": config.BgmTvAutoTracker.APP_SECRET,
},
) as auth_resp:
auth_time = dateutil.parser.parse(
auth_resp.headers["Date"]).timestamp()
auth_response = AuthResponse.parse_obj(await auth_resp.json())
async with await aio_client.get(
f"https://mirror.api.bgm.rin.cat/user/{auth_response.user_id}"
) as user_info_resp:
user_info = UserInfo.parse_obj(await user_info_resp.json())
user = Me(auth_time=auth_time, **auth_response.dict())
insert_stmt = sa.insert(sa.UserToken)
query = insert_stmt.on_duplicate_key_update(
**preserve_fields(
insert_stmt,
"token_type",
"expires_in",
"auth_time",
"access_token",
"refresh_token",
"scope",
"username",
"nickname",
"usergroup",
), )
await db.execute(
query,
{
"user_id": user.user_id,
"token_type": user.token_type,
"expires_in": user.expires_in,
"auth_time": user.auth_time,
"access_token": user.access_token,
"refresh_token": user.refresh_token,
"scope": auth_response.scope or "",
"username": user_info.username,
"nickname": user_info.nickname,
"usergroup": user_info.usergroup.value,
},
)
session = await new_session(user_id=auth_response.user_id, redis=redis)
response = JSONResponse({"api_key": session.api_key})
response.set_cookie(cookie_scheme.model.name, session.api_key)
return response
except aiohttp.ServerTimeoutError:
return JSONResponse(
content={"detail": f"connect to bgm.tv timeout"},
status_code=HTTP_503_SERVICE_UNAVAILABLE,
)
except (
json.decoder.JSONDecodeError,
ValidationError,
ConnectionError,
aiohttp.ServerConnectionError,
):
return redirect_response
async def deepfry(img: Image,
*,
token: str = None,
url_base: str = 'westcentralus',
session: aiohttp.ClientSession = None,
type=DeepfryTypes.RED) -> Image:
"""
Deepfry an image.
img: PIL.Image - Image to deepfry.
[token]: str - Token to use for Microsoft facial recognition API. If this is not supplied, lens flares will not be added.
[url_base]: str = 'westcentralus' - API base to use. Only needed if your key's region is not `westcentralus`.
[session]: aiohttp.ClientSession - Optional session to use with API requests. If provided, may provide a bit more speed.
Returns: PIL.Image - Deepfried image.
"""
img = img.copy().convert('RGB')
if type not in DeepfryTypes:
raise ValueError(
f'Unknown deepfry type "{type}", expected a value from deeppyer.DeepfryTypes'
)
if token:
req_url = f'https://{url_base}.api.cognitive.microsoft.com/face/v1.0/detect?returnFaceId=false&returnFaceLandmarks=true' # WHY THE F**K IS THIS SO LONG
headers = {
'Content-Type': 'application/octet-stream',
'Ocp-Apim-Subscription-Key': token,
'User-Agent': 'DeepPyer/1.0'
}
b = BytesIO()
img.save(b, 'jpeg')
b.seek(0)
if session:
async with session.post(req_url, headers=headers,
data=b.read()) as r:
face_data = await r.json()
else:
async with aiohttp.ClientSession() as s, s.post(
req_url, headers=headers, data=b.read()) as r:
face_data = await r.json()
if 'error' in face_data:
err = face_data['error']
code = err.get('code', err.get('statusCode'))
msg = err['message']
raise Exception(
f'Error with Microsoft Face Recognition API\n{code}: {msg}')
if face_data:
landmarks = face_data[0]['faceLandmarks']
# Get size and positions of eyes, and generate sizes for the flares
eye_left_width = math.ceil(landmarks['eyeLeftInner']['x'] -
landmarks['eyeLeftOuter']['x'])
eye_left_height = math.ceil(landmarks['eyeLeftBottom']['y'] -
landmarks['eyeLeftTop']['y'])
eye_left_corner = (landmarks['eyeLeftOuter']['x'],
landmarks['eyeLeftTop']['y'])
flare_left_size = eye_left_height if eye_left_height > eye_left_width else eye_left_width
flare_left_size *= 4
eye_left_corner = tuple(
math.floor(x - flare_left_size / 2.5 + 5)
for x in eye_left_corner)
eye_right_width = math.ceil(landmarks['eyeRightOuter']['x'] -
landmarks['eyeRightInner']['x'])
eye_right_height = math.ceil(landmarks['eyeRightBottom']['y'] -
landmarks['eyeRightTop']['y'])
eye_right_corner = (landmarks['eyeRightInner']['x'],
landmarks['eyeRightTop']['y'])
flare_right_size = eye_right_height if eye_right_height > eye_right_width else eye_right_width
flare_right_size *= 4
eye_right_corner = tuple(
math.floor(x - flare_right_size / 2.5 + 5)
for x in eye_right_corner)
# Crush image to hell and back
img = img.convert('RGB')
width, height = img.width, img.height
img = img.resize((int(width**.75), int(height**.75)),
resample=Image.LANCZOS)
img = img.resize((int(width**.88), int(height**.88)),
resample=Image.BILINEAR)
img = img.resize((int(width**.9), int(height**.9)), resample=Image.BICUBIC)
img = img.resize((width, height), resample=Image.BICUBIC)
img = ImageOps.posterize(img, 4)
# Generate red and yellow overlay for classic deepfry effect
r = img.split()[0]
r = ImageEnhance.Contrast(r).enhance(2.0)
r = ImageEnhance.Brightness(r).enhance(1.5)
if type == DeepfryTypes.RED:
r = ImageOps.colorize(r, Colours.RED, Colours.YELLOW)
elif type == DeepfryTypes.BLUE:
r = ImageOps.colorize(r, Colours.BLUE, Colours.WHITE)
# Overlay red and yellow onto main image and sharpen the hell out of it
img = Image.blend(img, r, 0.75)
img = ImageEnhance.Sharpness(img).enhance(100.0)
if token and face_data:
# Copy and resize flares
flare = Image.open('./deeppyer/flare.png')
flare_left = flare.copy().resize((flare_left_size, ) * 2,
resample=Image.BILINEAR)
flare_right = flare.copy().resize((flare_right_size, ) * 2,
resample=Image.BILINEAR)
del flare
img.paste(flare_left, eye_left_corner, flare_left)
img.paste(flare_right, eye_right_corner, flare_right)
return img
class ApiInstance(object):
def __init__(
self,
base_url: str,
timeout: int = 5000,
headers: dict = {},
*,
logger: logging.Logger
):
self.base_url = base_url
self.headers = headers
self.timeout = timeout
self.session = ClientSession(timeout=ClientTimeout(total=timeout))
self.logger = logger
async def __aenter__(self) -> "ApiInstance":
return self
async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None:
await self.close()
def check_result(self, res: Any) -> Any:
if res == "":
return None
return json_loads_attrs(res)
async def call(self, method: str, prefix: str, data: dict, headers: dict = {}):
new_headers = CIMultiDict()
new_headers.update(self.headers)
new_headers.update(headers)
data_binary = json.dumps(data).encode("utf-8")
# TODO: this is too much code duplication but I cannot think of
# a way outside macros that could abstract async with block
# and sadly there are no macro in python
if method == "get":
async with self.session.get(
self.base_url + "/" + prefix, headers=new_headers
) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "post":
async with self.session.post(
self.base_url + "/" + prefix, data=data_binary, headers=new_headers
) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "put":
async with self.session.put(
self.base_url + "/" + prefix, data=data_binary, headers=new_headers
) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "patch":
async with self.session.patch(
self.base_url + "/" + prefix, data=data_binary, headers=new_headers
) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
elif method == "delete":
async with self.session.delete(
self.base_url + "/" + prefix, headers=new_headers
) as resp:
await check_response(resp, self.logger)
res = await resp.text()
return self.check_result(res)
async def post(self, prefix: str, data: dict, headers: dict = {}) -> Any:
try:
return await self.call("post", prefix, data, headers)
except PapieaBaseException as papiea_exception:
raise papiea_exception
except ApiException as api_exception:
raise api_exception
except Exception as e:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call("post", prefix, data, headers)
async def put(self, prefix: str, data: dict, headers: dict = {}) -> Any:
try:
return await self.call("put", prefix, data, headers)
except PapieaBaseException as papiea_exception:
raise papiea_exception
except ApiException as api_exception:
raise api_exception
except Exception as e:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call("put", prefix, data, headers)
async def patch(self, prefix: str, data: dict, headers: dict = {}) -> Any:
try:
return await self.call("patch", prefix, data, headers)
except PapieaBaseException as papiea_exception:
raise papiea_exception
except ApiException as api_exception:
raise api_exception
except Exception as e:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call("patch", prefix, data, headers)
async def get(self, prefix: str, headers: dict = {}) -> Any:
try:
return await self.call("get", prefix, {}, headers)
except PapieaBaseException as papiea_exception:
raise papiea_exception
except ApiException as api_exception:
raise api_exception
except Exception as e:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call("get", prefix, {}, headers)
async def delete(self, prefix: str, headers: dict = {}) -> Any:
try:
return await self.call("delete", prefix, {}, headers)
except PapieaBaseException as papiea_exception:
raise papiea_exception
except ApiException as api_exception:
raise api_exception
except Exception as e:
self.logger.debug("RENEWING SESSION")
await self.renew_session()
return await self.call("delete", prefix, {}, headers)
async def close(self):
await self.session.close()
async def renew_session(self):
await self.close()
self.session = ClientSession(timeout=ClientTimeout(total=self.timeout))
class HttpTransport(BaseOutboundTransport):
"""Http outbound transport class."""
schemes = ("http", "https")
def __init__(self) -> None:
"""Initialize an `HttpTransport` instance."""
super().__init__()
self.client_session: ClientSession = None
self.connector: TCPConnector = None
self.logger = logging.getLogger(__name__)
async def start(self):
"""Start the transport."""
session_args = {}
self.connector = TCPConnector(limit=200, limit_per_host=50)
if self.collector:
session_args["trace_configs"] = [
StatsTracer(self.collector, "outbound-http:")
]
session_args["cookie_jar"] = DummyCookieJar()
session_args["connector"] = self.connector
self.client_session = ClientSession(**session_args)
return self
async def stop(self):
"""Stop the transport."""
await self.client_session.close()
self.client_session = None
async def handle_message(
self,
profile: Profile,
payload: Union[str, bytes],
endpoint: str,
metadata: dict = None,
api_key: str = None,
):
"""
Handle message from queue.
Args:
profile: the profile that produced the message
payload: message payload in string or byte format
endpoint: URI endpoint for delivery
metadata: Additional metadata associated with the payload
"""
if not endpoint:
raise OutboundTransportError("No endpoint provided")
headers = metadata or {}
if api_key is not None:
headers["x-api-key"] = api_key
if isinstance(payload, bytes):
if profile.settings.get("emit_new_didcomm_mime_type"):
headers["Content-Type"] = DIDCOMM_V1_MIME_TYPE
else:
headers["Content-Type"] = DIDCOMM_V0_MIME_TYPE
else:
headers["Content-Type"] = "application/json"
self.logger.debug(
"Posting to %s; Data: %s; Headers: %s", endpoint, payload, headers
)
async with self.client_session.post(
endpoint, data=payload, headers=headers
) as response:
if response.status < 200 or response.status > 299:
raise OutboundTransportError(
(
f"Unexpected response status {response.status}, "
f"caused by: {response.reason}"
)
)
def route(request):
session = yield from get_session(request)
if 'uid' in session:
uid = session['uid']
else:
return toolbox.javaify(403, "forbidden")
if request.content_type != "multipart/form-data":
return toolbox.javaify(400, "content type error")
query_parameters = request.rel_url.query
if "dir" in query_parameters:
directory = query_parameters["dir"]
else:
return toolbox.javaify(400, "miss parameter")
with (yield from request.app['pool']) as connect:
cursor = yield from connect.cursor()
directory_id = yield from fs.directory_exists(cursor, uid, directory)
if not directory_id:
yield from cursor.close()
connect.close()
return toolbox.javaify(400, "target error")
try:
reader = yield from request.multipart()
part = yield from reader.next()
except Exception as e:
print(e)
yield from cursor.close()
connect.close()
return toolbox.javaify(400, "bad request")
if part is None:
yield from cursor.close()
connect.close()
return toolbox.javaify(400, "bad request")
try:
file_name = re.search(r'filename="([^"]+)"',
part.headers["Content-Disposition"]).group(1)
file_extension = os.path.splitext(file_name)[-1][1:]
except:
yield from cursor.close()
connect.close()
return toolbox.javaify(400, "bad request")
if fs.name_illegal(file_name):
return toolbox.javaify(400, "name illegal")
file_meta = yield from fs.file_query(cursor, directory_id, [file_name])
if file_meta and file_meta[0]["type"] == "directory" and file_meta[0][
"status"] == 1:
yield from cursor.close()
connect.close()
return toolbox.javaify(400, "directory exists")
now = datetime.datetime.now()
# size = 0
# temp_path = os.path.join(request.app["temp_dir"],str(uuid.uuid4()))
# f = open(temp_path,'wb')
# md5 = hashlib.md5()
# while True:
# try:
# chunk = yield from part.read_chunk() # 8192 bytes by default
# except:
# yield from cursor.close()
# connect.close()
# return toolbox.javaify(415,"unsupported media type")
# if not chunk:
# md5 = md5.hexdigest()
# f.close()
# file_type = fs.mime_detect(temp_path)
# break
# else:
# size += len(chunk)
# f.write(chunk)
# md5.update(chunk)
session = ClientSession()
boundary = '----------{}'.format(uuid.uuid4().hex)
content_type = 'multipart/form-data; boundary={}'.format(boundary)
try:
response = yield from session.post(
'http://up-z2.qiniu.com',
data=rewrite(uid, part, boundary),
headers={'Content-Type': content_type})
json_back = yield from response.text()
yield from session.close()
json_back = json.loads(json_back)
file_type = json_back["type"]
size = json_back["size"]
md5 = json_back["key"].split("/")[-1]
except Exception as error:
print(error)
yield from cursor.close()
connect.close()
return toolbox.javaify(500, "something wrong")
if file_meta and file_meta[0]["type"] == "directory" and file_meta[0][
"status"] == 0:
yield from fs.file_delete(cursor, [file_meta[0]["id"]])
request.app.loop.create_task(
fs.file_delete_async(request.app['pool'],
[file_meta[0]["id"]]))
file_meta = None
if not file_meta:
file_id = yield from fs.file_create(
cursor, {
"directory": directory_id,
"name": file_name,
"type": file_type,
"modify": toolbox.time_str(now),
"size": size,
"md5": md5,
})
else:
file_id = file_meta[0]["id"]
yield from fs.file_modify(
cursor, file_id, {
"type": file_type,
"modify": toolbox.time_str(now),
"size": size,
"md5": md5,
"status": 1
})
# yield from fs.version_sync(cursor,file_id)
yield from fs.file_modify(cursor, directory_id,
{"modify": toolbox.time_str(now)})
# if not file_meta or md5 != file_meta[0]['md5']:
# request.app.loop.create_task(oss.transmit(temp_path,"{}/{}".format(str(uid).zfill(8),md5)))
# pass
yield from connect.commit()
yield from cursor.close()
connect.close()
return toolbox.javaify(
200, "success", {
"name": file_name,
"type": file_type,
"extension": file_extension,
"modify": toolbox.time_utc(now),
"owner": "self",
"size": size,
"source": mask.generate(uid, md5, file_extension)
})
class AdminServer(BaseAdminServer):
"""Admin HTTP server class."""
def __init__(
self,
host: str,
port: int,
context: InjectionContext,
outbound_message_router: Coroutine,
):
"""
Initialize an AdminServer instance.
Args:
host: Host to listen on
port: Port to listen on
"""
self.app = None
self.host = host
self.port = port
self.loaded_modules = []
self.webhook_queue = None
self.webhook_retries = 5
self.webhook_session: ClientSession = None
self.webhook_targets = {}
self.webhook_task = None
self.webhook_processor: TaskProcessor = None
self.websocket_queues = {}
self.site = None
self.context = context.start_scope("admin")
self.responder = AdminResponder(outbound_message_router,
self.send_webhook)
self.context.injector.bind_instance(BaseResponder, self.responder)
async def make_application(self) -> web.Application:
"""Get the aiohttp application instance."""
middlewares = []
admin_api_key = self.context.settings.get("admin.admin_api_key")
admin_insecure_mode = self.context.settings.get(
"admin.admin_insecure_mode")
# admin-token and admin-token are mutually exclusive and required.
# This should be enforced during parameter parsing but to be sure,
# we check here.
assert admin_insecure_mode or admin_api_key
assert not (admin_insecure_mode and admin_api_key)
# If admin_api_key is None, then admin_insecure_mode must be set so
# we can safely enable the admin server with no security
if admin_api_key:
@web.middleware
async def check_token(request, handler):
header_admin_api_key = request.headers.get("x-api-key")
if not header_admin_api_key:
raise web.HTTPUnauthorized()
if admin_api_key == header_admin_api_key:
return await handler(request)
else:
raise web.HTTPUnauthorized()
middlewares.append(check_token)
stats: Collector = await self.context.inject(Collector, required=False)
if stats:
@web.middleware
async def collect_stats(request, handler):
handler = stats.wrap_coro(
handler, [handler.__qualname__, "any-admin-request"])
return await handler(request)
middlewares.append(collect_stats)
app = web.Application(middlewares=middlewares)
app["request_context"] = self.context
app["outbound_message_router"] = self.responder.send
app.add_routes([
web.get("/", self.redirect_handler),
web.get("/modules", self.modules_handler),
web.get("/status", self.status_handler),
web.post("/status/reset", self.status_reset_handler),
web.get("/ws", self.websocket_handler),
])
await register_module_routes(app)
for protocol_module_path in self.context.settings.get(
"external_protocols", []):
try:
routes_module = ClassLoader.load_module(
f"{protocol_module_path}.routes")
await routes_module.register(app)
except Exception as e:
raise ConfigError(
f"Failed to load external protocol module '{protocol_module_path}'."
) from e
cors = aiohttp_cors.setup(
app,
defaults={
"*":
aiohttp_cors.ResourceOptions(
allow_credentials=True,
expose_headers="*",
allow_headers="*",
allow_methods="*",
)
},
)
for route in app.router.routes():
cors.add(route)
setup_aiohttp_apispec(app=app,
title="Aries Cloud Agent",
version="v1",
swagger_path="/api/doc")
app.on_startup.append(self.on_startup)
return app
async def start(self) -> None:
"""
Start the webserver.
Raises:
AdminSetupError: If there was an error starting the webserver
"""
self.app = await self.make_application()
runner = web.AppRunner(self.app)
await runner.setup()
self.site = web.TCPSite(runner, host=self.host, port=self.port)
try:
await self.site.start()
except OSError:
raise AdminSetupError("Unable to start webserver with host " +
f"'{self.host}' and port '{self.port}'\n")
async def stop(self) -> None:
"""Stop the webserver."""
for queue in self.websocket_queues.values():
queue.stop()
if self.site:
await self.site.stop()
self.site = None
if self.webhook_queue:
self.webhook_queue.stop()
self.webhook_queue = None
if self.webhook_session:
await self.webhook_session.close()
self.webhook_session = None
async def on_startup(self, app: web.Application):
"""Perform webserver startup actions."""
@docs(tags=["server"], summary="Fetch the list of loaded modules")
@response_schema(AdminModulesSchema(), 200)
async def modules_handler(self, request: web.BaseRequest):
"""
Request handler for the loaded modules list.
Args:
request: aiohttp request object
Returns:
The module list response
"""
return web.json_response({"result": self.loaded_modules})
@docs(tags=["server"], summary="Fetch the server status")
@response_schema(AdminStatusSchema(), 200)
async def status_handler(self, request: web.BaseRequest):
"""
Request handler for the server status information.
Args:
request: aiohttp request object
Returns:
The web response
"""
status = {}
collector: Collector = await self.context.inject(Collector,
required=False)
if collector:
status["timing"] = collector.results
return web.json_response(status)
@docs(tags=["server"], summary="Reset statistics")
@response_schema(AdminStatusSchema(), 200)
async def status_reset_handler(self, request: web.BaseRequest):
"""
Request handler for resetting the timing statistics.
Args:
request: aiohttp request object
Returns:
The web response
"""
collector: Collector = await self.context.inject(Collector,
required=False)
if collector:
collector.reset()
return web.json_response({})
async def redirect_handler(self, request: web.BaseRequest):
"""Perform redirect to documentation."""
raise web.HTTPFound("/api/doc")
async def websocket_handler(self, request):
"""Send notifications to admin client over websocket."""
ws = web.WebSocketResponse()
await ws.prepare(request)
socket_id = str(uuid.uuid4())
queue = await self.context.inject(BaseOutboundMessageQueue)
try:
self.websocket_queues[socket_id] = queue
await queue.enqueue({
"topic": "settings",
"payload": {
"label":
self.context.settings.get("default_label"),
"endpoint":
self.context.settings.get("default_endpoint"),
"no_receive_invites":
self.context.settings.get("admin.no_receive_invites",
False),
"help_link":
self.context.settings.get("admin.help_link"),
},
})
closed = False
while not closed:
try:
msg = await queue.dequeue(timeout=5.0)
if msg is None:
# we send fake pings because the JS client
# can't detect real ones
msg = {"topic": "ping"}
if ws.closed:
closed = True
if msg and not closed:
await ws.send_json(msg)
except asyncio.CancelledError:
closed = True
finally:
del self.websocket_queues[socket_id]
return ws
def add_webhook_target(self,
target_url: str,
topic_filter: Sequence[str] = None,
retries: int = None):
"""Add a webhook target."""
self.webhook_targets[target_url] = WebhookTarget(
target_url, topic_filter, retries)
def remove_webhook_target(self, target_url: str):
"""Remove a webhook target."""
if target_url in self.webhook_targets:
del self.webhook_targets[target_url]
async def send_webhook(self, topic: str, payload: dict):
"""Add a webhook to the queue, to send to all registered targets."""
if not self.webhook_queue:
self.webhook_queue = await self.context.inject(
BaseOutboundMessageQueue)
self.webhook_task = asyncio.ensure_future(self._process_webhooks())
await self.webhook_queue.enqueue((topic, payload))
async def _process_webhooks(self):
"""Continuously poll webhook queue and dispatch to targets."""
self.webhook_session = ClientSession()
self.webhook_processor = TaskProcessor(max_pending=5)
async for topic, payload in self.webhook_queue:
for queue in self.websocket_queues.values():
await queue.enqueue({"topic": topic, "payload": payload})
if self.webhook_targets:
targets = self.webhook_targets.copy()
for idx, target in targets.items():
if topic == "connections_activity":
# filter connections activity by default (only sent to sockets)
continue
if not target.topic_filter or topic in target.topic_filter:
retries = (self.webhook_retries if
target.retries is None else target.retries)
await self.webhook_processor.run_retry(
lambda pending: self._perform_send_webhook(
target.endpoint, topic, payload, pending.
attempts + 1),
ident=(target.endpoint, topic),
retries=retries,
)
self.webhook_queue.task_done()
async def _perform_send_webhook(self,
target_url: str,
topic: str,
payload: dict,
attempt: int = None):
"""Dispatch a webhook to a specific endpoint."""
full_webhook_url = f"{target_url}/topic/{topic}/"
attempt_str = f" (attempt {attempt})" if attempt else ""
LOGGER.debug("Sending webhook to : %s%s", full_webhook_url,
attempt_str)
async with self.webhook_session.post(full_webhook_url,
json=payload) as response:
if response.status < 200 or response.status > 299:
# raise Exception(f"Unexpected response status {response.status}")
raise Exception(f"Unexpected: target {target_url}\n"
f"full {full_webhook_url}\n"
f"response {response}")
async def complete_webhooks(self):
"""Wait for all pending webhooks to be dispatched, used in testing."""
if self.webhook_queue:
await self.webhook_queue.join()
self.webhook_queue.stop()
if self.webhook_processor:
await self.webhook_processor.wait_done()
class CrawlManager:
"""A simple class for managing crawls"""
def __init__(self) -> None:
self.redis: Redis = None
self.session: ClientSession = None
self.loop: AbstractEventLoop = None
self.depth: int = env('DEFAULT_DEPTH', type_=int, default=1)
self.same_domain_depth: int = env(
'DEFAULT_SAME_DOMAIN_DEPTH', type_=int, default=100
)
self.num_browsers: int = env('DEFAULT_NUM_BROWSERS', type_=int, default=2)
self.flock: str = env('DEFAULT_FLOCK', default='browsers')
self.shepherd_host: str = env(
'DEFAULT_SHEPHERD', default='http://shepherd:9020'
)
self.browser_api_url: str = f'{self.shepherd_host}/api'
self.pool: str = env('DEFAULT_POOL', default='')
self.scan_key: str = 'a:*:info'
self.container_environ: Dict[str, str] = {
'URL': 'about:blank',
'REDIS_URL': env('REDIS_URL', default=DEFAULT_REDIS_URL),
'WAIT_FOR_Q': '10',
'TAB_TYPE': 'CrawlerTab',
'CRAWL_NO_NETCACHE': '0',
'VNC_PASS': '******',
'IDLE_TIMEOUT': '',
'BEHAVIOR_API_URL': 'http://behaviors:3030',
'SCREENSHOT_API_URL': env('SCREENSHOT_API_URL'),
}
self.default_browser = None
if os.environ.get('DEBUG'):
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
async def startup(self) -> None:
"""Initialize the crawler manager's redis connection and
http session used to make requests to shepherd
"""
self.loop = get_event_loop()
self.redis = await init_redis(
env('REDIS_URL', default=DEFAULT_REDIS_URL), self.loop
)
self.session = ClientSession(
connector=TCPConnector(
resolver=AsyncResolver(loop=self.loop), loop=self.loop
),
json_serialize=partial(json.dumps, ensure_ascii=False),
loop=self.loop,
)
async def shutdown(self) -> None:
"""Closes the redis connection and http session"""
try:
self.redis.close()
await self.redis.wait_closed()
except Exception:
pass
try:
await self.session.close()
except Exception:
pass
def new_crawl_id(self) -> str:
"""Creates an id for a new crawl
:return: The new id for a crawl
"""
return uuid.uuid4().hex[-12:]
async def create_new(
self, crawl_request: CreateCrawlRequest
) -> Dict[str, Union[bool, str]]:
"""Creates a new crawl
:param crawl_request: The body of the api request
for the /crawls endpoint
:return: A dictionary indicating success and the id of the new crawl
"""
crawl_id = self.new_crawl_id()
crawl = Crawl(crawl_id, self)
return await crawl.init_crawl(crawl_request)
async def load_crawl(self, crawl_id: str) -> Crawl:
"""Returns the crawl information for the supplied crawl id
:param crawl_id: The id of the crawl to load
:return: The information about a crawl
"""
crawl = Crawl(crawl_id, self)
data = await self.redis.hgetall(crawl.info_key)
if not data:
raise HTTPException(404, detail='crawl not found')
crawl.model = CrawlInfo(**data)
return crawl
async def get_full_crawl_info(self, crawl_id: str) -> Dict:
crawl = await self.load_crawl(crawl_id)
info, urls = await aio_gather(
crawl.get_info(count_urls=False), crawl.get_info_urls(), loop=self.loop
)
return dict(**info, **urls, success=True)
async def do_request(self, url_path: str, post_data: Optional[Dict] = None) -> Dict:
"""Makes an HTTP post request to the supplied URL/path
:param url_path: The URL or path for the post request
:param post_data: Optional post request body
:return: The response body
"""
try:
url = self.browser_api_url + url_path
async with self.session.post(url, json=post_data) as res:
res = await res.json()
logger.debug(str(res))
return res
except Exception as e:
text = str(e)
logger.debug(text)
raise HTTPException(400, text)
async def get_all_crawls(self) -> Dict[str, List[Dict]]:
"""Returns crawl info for all crawls
:return: The list of all crawl info
"""
all_infos = []
async for key in self.redis.iscan(match=self.scan_key):
_, crawl_id, _2 = key.split(':', 2)
try:
# crawl = Crawl(crawl_id, self)
crawl = await self.load_crawl(crawl_id)
info = await crawl.get_info()
all_infos.append(info)
except HTTPException:
continue
return {'crawls': all_infos}
async def request_flock(self, opts: Dict) -> Dict:
"""Requests a flock from shepherd using the supplied options
:param opts: The options for the requested flock
:return: The response from shepherd
"""
response = await self.do_request(
f'/flock/request/{self.flock}?pool={self.pool}', opts
)
return response
async def start_flock(self, reqid: str) -> Dict:
"""Requests that shepherd start the flock identified by the
supplied request id
:param reqid: The request id of the flock to be started
:return: The response from shepherd
"""
response = await self.do_request(
f'/flock/start/{reqid}', {'environ': {'REQ_ID': reqid}}
)
return response
async def stop_flock(self, reqid: str) -> Dict:
"""Requests that shepherd stop, but not remove, the flock
identified by the supplied request id
:param reqid: The request id of the flock to be stopped
:return: The response from shepherd
"""
response = await self.do_request(f'/flock/stop/{reqid}')
return response
async def remove_flock(self, reqid: str) -> Dict:
"""Requests that shepherd stop and remove the flock
identified by the supplied request id
:param reqid: The request id of the flock to be stopped
:return: The response from shepherd
"""
response = await self.do_request(f'/flock/remove/{reqid}')
return response
class Tio:
def __init__(
self,
*,
session: Optional[ClientSession] = None,
loop: Optional[asyncio.AbstractEventLoop] = None,
store_languages: Optional[bool] = True,
) -> None:
self._store_languages = store_languages
self.API_URL = "https://tio.run/cgi-bin/run/api/"
self.LANGUAGES_URL = "https://tio.run/languages.json"
self.languages = []
if loop:
self.loop = loop
else:
try:
self.loop = asyncio.get_running_loop()
except RuntimeError:
self.loop = asyncio.get_event_loop()
if session:
self.session = session
else:
self.session = None
if self.loop.is_running():
self.loop.create_task(self._initialize())
else:
self.loop.run_until_complete(self._initialize())
return None
async def __aenter__(self) -> Tio:
await self._initialize()
return self
async def __aexit__(self, *_) -> None:
await self.close()
async def close(self) -> None:
await self.session.close()
async def _initialize(self) -> None:
if not self.session:
self.session = ClientSession()
if self._store_languages:
async with self.session.get(self.LANGUAGES_URL) as r:
if r.ok:
data = await r.json()
self.languages = list(data.keys())
return None
def _format_payload(self, name: str, obj: Union[list, str]) -> bytes:
if not obj:
return b''
elif isinstance(obj, list):
content = ['V' + name, str(len(obj))] + obj
return bytes('\x00'.join(content) + '\x00', encoding='utf-8')
else:
return bytes(
f"F{name}\x00{len(bytes(obj, encoding='utf-8'))}\x00{obj}\x00",
encoding='utf-8')
async def execute(
self,
code: str,
*,
language: str,
inputs: Optional[str] = "",
compiler_flags: Optional[list] = [],
Cl_options: Optional[list] = [],
arguments: Optional[list] = [],
) -> Optional[TioResponse]:
if language not in self.languages:
match = [l for l in self.languages if language in l]
if match:
language = match[0]
data = {
"lang": [language],
".code.tio": code,
".input.tio": inputs,
"TIO_CFLAGS": compiler_flags,
"TIO_OPTIONS": Cl_options,
"args": arguments,
}
bytes_ = b''.join(map(self._format_payload, data.keys(),
data.values())) + b'R'
data = compress(bytes_, 9)[2:-4]
async with self.session.post(self.API_URL, data=data) as r:
if r.ok:
data = await r.read()
data = data.decode("utf-8")
if re.search(
r"The language ?'.+' ?could not be found on the server.",
data):
raise LanguageNotFound(data[16:])
else:
return TioResponse(data, language)
else:
raise ApiError(f"Error {r.status}, {r.reason}")
class SmartThingsTV:
def __init__(
self,
api_key: str,
device_id: str,
refresh_status: bool = True,
session: Optional[ClientSession] = None,
):
"""Initialize SmartThingsTV."""
self._api_key = api_key
self._device_id = device_id
self._refresh_status = refresh_status
if session:
self._session = session
self._managed_session = False
else:
self._session = ClientSession()
self._managed_session = True
self._device_name = None
self._state = STATE_OFF
self._muted = False
self._volume = 10
self._source_list = None
self._prev_source = None
self._source = None
self._prev_channel = ""
self._channel = ""
self._prev_channel_name = ""
self._channel_name = ""
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
pass
@property
def api_key(self) -> str:
"""Return currently api_key."""
return self._api_key
@property
def device_id(self) -> str:
"""Return currently device_id."""
return self._device_id
@property
def device_name(self) -> str:
"""Return currently device_name."""
return self._device_name
@property
def state(self) -> str:
"""Return currently state."""
return self._state
@property
def muted(self) -> bool:
"""Return currently muted state."""
return self._muted
@property
def volume(self) -> int:
"""Return currently volume."""
return self._volume
@property
def source(self) -> str:
"""Return currently source."""
return self._source
@property
def channel(self) -> str:
"""Return currently channel."""
return self._channel
@property
def channel_name(self) -> str:
"""Return currently channel_name."""
return self._channel_name
@property
def source_list(self):
"""Return currently channel_name."""
return self._source_list
def set_application(self, appid):
if self._refresh_status:
self._channel = ""
self._channel_name = appid
@staticmethod
async def get_devices_list(api_key, session: ClientSession, device_label=""):
"""Get list of available devices"""
result = {}
async with session.get(
API_DEVICES, headers=_headers(api_key), raise_for_status=True,
) as resp:
device_list = await resp.json()
if device_list:
_LOGGER.debug("SmartThings available devices: %s", str(device_list))
for k in device_list.get("items", []):
device_id = k.get("deviceId", "")
device_type = k.get("type", "")
device_type_id = k.get("deviceTypeId", "")
if device_id and (
device_type_id == DEVICE_TYPEID_OCF
or device_type == DEVICE_TYPE_OCF
):
label = k.get("label", "")
if device_label == "" or (label == device_label and label != ""):
result.setdefault(device_id, {})["name"] = k.get("name", "")
result.setdefault(device_id, {})["label"] = label
_LOGGER.info("SmartThings discovered TV devices: %s", str(result))
return result
@Throttle(MIN_TIME_BETWEEN_UPDATES)
async def _device_refresh(self, **kwargs):
"""Refresh device status on SmartThings"""
device_id = self._device_id
if not device_id:
return
api_device = f"{API_DEVICES}/{device_id}"
api_command = f"{api_device}/commands"
if self._refresh_status:
async with self._session.post(
api_command,
headers=_headers(self._api_key),
data=COMMAND_REFRESH,
raise_for_status=True,
) as resp:
await resp.json()
async def async_device_health(self):
"""Check device availability"""
device_id = self._device_id
if not device_id:
return False
api_device = f"{API_DEVICES}/{device_id}"
api_device_health = f"{api_device}/health"
# this get the real status of the device
async with self._session.get(
api_device_health, headers=_headers(self._api_key), raise_for_status=True,
) as resp:
health = await resp.json()
_LOGGER.debug(health)
if health["state"] == "ONLINE":
return True
return False
async def async_device_update(self):
"""Query device status on SmartThing"""
device_id = self._device_id
if not device_id:
return
api_device = f"{API_DEVICES}/{device_id}"
api_device_status = f"{api_device}/states"
# not used, just for reference
api_device_main_status = f"{api_device}/components/main/status"
is_online = await self.async_device_health()
if is_online:
self._state = STATE_ON
else:
self._state = STATE_OFF
return
await self._device_refresh()
async with self._session.get(
api_device_status, headers=_headers(self._api_key), raise_for_status=True,
) as resp:
data = await resp.json()
_LOGGER.debug(data)
# device_state = data['main']['switch']['value']
device_volume = data["main"]["volume"]["value"]
device_volume = int(device_volume) / 100
device_muted = data["main"]["mute"]["value"]
device_all_sources = json.loads(data["main"]["supportedInputSources"]["value"])
device_source = data["main"]["inputSource"]["value"]
device_tv_chan = data["main"]["tvChannel"]["value"]
device_tv_chan_name = data["main"]["tvChannelName"]["value"]
self._volume = device_volume
self._source_list = device_all_sources
if device_muted == "mute":
self._muted = True
else:
self._muted = False
if (
self._prev_source != device_source
or self._prev_channel != device_tv_chan
or self._prev_channel_name != device_tv_chan_name
):
self._source = device_source
self._prev_source = device_source
# if the status is not refreshed this info may become not reliable
if self._refresh_status:
self._channel = device_tv_chan
self._prev_channel = device_tv_chan
self._channel_name = device_tv_chan_name
self._prev_channel_name = device_tv_chan_name
async def async_send_command(self, cmdtype, command=""):
"""Send a command too the device"""
device_id = self._device_id
if not device_id:
return
api_device = f"{API_DEVICES}/{device_id}"
api_command = f"{api_device}/commands"
datacmd = None
if cmdtype == "turn_off": # turns off
datacmd = COMMAND_POWER_OFF
elif cmdtype == "turn_on": # turns on
datacmd = COMMAND_POWER_ON
elif cmdtype == "setvolume": # sets volume
cmdargs = ARGS_SET_VOLUME.format(command)
datacmd = COMMAND_SET_VOLUME + cmdargs
elif cmdtype == "stepvolume": # steps volume up or down
if command == "up":
datacmd = COMMAND_VOLUME_UP
elif command == "down":
datacmd = COMMAND_VOLUME_DOWN
elif cmdtype == "audiomute": # mutes audio
if command == "on":
datacmd = COMMAND_MUTE
elif command == "off":
datacmd = COMMAND_UNMUTE
elif cmdtype == "selectchannel": # changes channel
cmdargs = ARGS_SET_CHANNEL.format(command)
datacmd = COMMAND_SET_CHANNEL + cmdargs
elif cmdtype == "stepchannel": # steps channel up or down
if command == "up":
datacmd = COMMAND_CHANNEL_UP
elif command == "down":
datacmd = COMMAND_CHANNEL_DOWN
elif cmdtype == "selectsource": # changes source
cmdargs = ARGS_SET_SOURCE.format(command)
datacmd = COMMAND_SET_SOURCE + cmdargs
# set property to reflect new changes
self._source = command
self._channel = ""
self._channel_name = ""
if datacmd:
async with self._session.post(
api_command,
headers=_headers(self._api_key),
data=datacmd,
raise_for_status=True,
) as resp:
await resp.json()
await self._device_refresh()
class Client(object):
def __init__(self, url, dumper=None, loop=None):
self.url = url
self.dumper = dumper
if not loop:
loop = asyncio.get_event_loop()
if not self.dumper:
self.dumper = json.dumps
self.client = ClientSession(
loop=loop,
headers={'content-type': 'application/json'})
def __del__(self):
self.client.close()
def __encode(self, method, params=None, id=None):
try:
data = self.dumper({
"jsonrpc": "2.0",
"id": id,
"method": method,
"params": params
})
except Exception as e:
raise Exception("Can not encode: {}".format(e))
return data
@asyncio.coroutine
def call(self, method, params=None, id=None, schem=None):
if not id:
id = uuid4().hex
try:
resp = yield from self.client.post(
self.url, data=self.__encode(method, params, id))
except Exception as err:
raise Exception(err)
if 200 != resp.status:
raise InvalidResponse(
"Error, server retunrned: {status}".format(status=resp.status))
try:
data = yield from resp.json()
except Exception as err:
raise InvalidResponse(err)
try:
validate(data, ERR_JSONRPC20)
return Response(**data)
except ValidationError:
# Passing data to validate response.
# Good if does not valid to ERR_JSONRPC20 object.
pass
except Exception as err:
raise InvalidResponse(err)
try:
validate(data, RSP_JSONRPC20)
if id != data['id']:
raise InvalidResponse(
"Rsponse id {local} not equal {remote}".format(
local=id, remote=data['id']))
except Exception as err:
raise InvalidResponse(err)
if schem:
try:
validate(data['result'], schem)
except ValidationError as err:
raise InvalidResponse(err)
except Exception as err:
raise InternalError(err)
return Response(**data)
class MobPushApi(MobPushBase):
"""
Mob Push SDK
:doc: https://www.mob.com/wiki/detailed/?wiki=MobPushRestAPIfenlei1333&id=136
"""
g_session: Optional[ClientSession] = None
async def push_to_all(self, content: str):
"""
推送给所有设备
:doc: https://www.mob.com/wiki/detailed/?wiki=MobPushRestAPIfenlei1333&id=136
:param content: 推送内容
:return:
"""
app_key = self._app_key
data = {
"source": "webapi",
"appkey": app_key,
"pushTarget": {
"target": 1, # 广播
},
"pushNotify": {
"plats": [1],
"content": content,
"type": 1,
},
}
await self._do_push(data, app_key)
async def push(self, device: str, content: str):
"""
推送一条消息
:doc: https://www.mob.com/wiki/detailed/?wiki=MobPushRestAPIfenlei1333&id=136
:return:
"""
app_key = self._app_key
data = {
"source": "webapi",
"appkey": app_key,
"pushTarget": {
"target": 4,
"rids": [device],
},
"pushNotify": {
"plats": [1],
"content": content,
"type": 1,
},
}
await self._do_push(data, app_key)
async def _do_push(self, data: dict, app_key: str):
if self.g_session is None:
self.g_session = ClientSession()
sign = await self.sign_fun(data)
headers = {"key": app_key, "sign": sign}
async with self.g_session.post(
self.s_push_url, data=data, headers=headers
) as ret:
if 200 <= ret.status < 300:
j = await ret.json()
self._log.bind(data=data, ret=j).info("request mob push success")
else:
self._log.bind(data=data).error("request mob push failed")
async def sign_fun(self, data: Optional[dict]) -> str:
"""
计算 mob 的签名
:param data:
:return:
"""
return self.compute_sign(data)
