async def _clickablePoint(self) -> Dict[str, float]: # noqa: C901
result = None
try:
result = await self._client.send(
'DOM.getContentQuads', {
'objectId': self._remoteObject.get('objectId'),
})
except Exception as e:
debugError(logger, e)
if not result or not result.get('quads'):
raise ElementHandleError(
'Node is either not visible or not an HTMLElement')
quads = []
for _quad in result.get('quads'):
_q = self._fromProtocolQuad(_quad)
if _computeQuadArea(_q) > 1:
quads.append(_q)
if not quads:
raise ElementHandleError(
'Node is either not visible or not an HTMLElement')
quad = quads[0]
x = 0
y = 0
for point in quad:
x += point['x']
y += point['y']
return {'x': x / 4, 'y': y / 4}
def __init__(self, client: 'CDPSession', url: str, consoleAPICalled: Callable[([str, List[JSHandle]], None)], exceptionThrown: Callable[([Dict], None)]) -> None:
super().__init__()
self._client = client
self._url = url
self._loop = client._loop
self._executionContextPromise = self._loop.create_future()
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
return None
def onExecutionContentCreated(event: Dict) -> None:
nonlocal jsHandleFactory
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
return JSHandle(executionContext, client, remoteObject)
executionContext = ExecutionContext(
client, event['context'], jsHandleFactory)
self._executionContextCallback(executionContext)
self._client.on('Runtime.executionContextCreated',
onExecutionContentCreated)
try:
self._client.send('Runtime.enable', {
})
except Exception as e:
debugError(logger, e)
def onConsoleAPICalled(event: Dict) -> None:
args = []
for arg in event.get('args', []):
args.append(jsHandleFactory(arg))
consoleAPICalled(event['type'], args)
self._client.on('Runtime.consoleAPICalled', onConsoleAPICalled)
self._client.on('Runtime.exceptionThrown', (lambda exception: exceptionThrown(
exception['exceptionDetails'])))
async def continue_(self, overrides: Dict = None) -> None:
"""Continue request with optional request overrides.
To use this method, request interception should be enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`. If request
interception is not enabled, raise ``NetworkError``.
``overrides`` can have the following fields:
* ``url`` (str): If set, the request url will be changed.
* ``method`` (str): If set, change the request method (e.g. ``GET``).
* ``postData`` (str): If set, change the post data or request.
* ``headers`` (dict): If set, change the request HTTP header.
"""
if overrides is None:
overrides = {}
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
opt = {'interceptionId': self._interceptionId}
opt.update(overrides)
try:
await self._client.send('Network.continueInterceptedRequest', opt)
except Exception as e:
debugError(logger, e)
async def _clickablePoint(self) -> Dict[str, float]: # noqa: C901
result = None
try:
result = await self._client.send('DOM.getContentQuads', {
'objectId': self._remoteObject.get('objectId'),
})
except Exception as e:
debugError(logger, e)
if not result or not result.get('quads'):
raise ElementHandleError(
'Node is either not visible or not an HTMLElement')
quads = []
for _quad in result.get('quads'):
_q = self._fromProtocolQuad(_quad)
if _computeQuadArea(_q) > 1:
quads.append(_q)
if not quads:
raise ElementHandleError(
'Node is either not visible or not an HTMLElement')
quad = quads[0]
x = 0
y = 0
for point in quad:
x += point['x']
y += point['y']
return {'x': x / 4, 'y': y / 4}
async def respond(self, response: Dict) -> None: # noqa: C901
"""Fulfills request with given response.
To use this, request interception should by enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`. Request
interception is not enabled, raise ``NetworkError``.
``response`` is a dictionary which can have the following fields:
* ``status`` (int): Response status code, defaults to 200.
* ``headers`` (dict): Optional response headers.
* ``contentType`` (str): If set, equals to setting ``Content-Type``
response header.
* ``body`` (str|bytes): Optional response body.
"""
if self._url.startswith('data:'):
return
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
if response.get('body') and isinstance(response['body'], str):
responseBody: Optional[bytes] = response['body'].encode('utf-8')
else:
responseBody = response.get('body')
responseHeaders = []
if response.get('headers'):
for key, value in response['headers'].items():
responseHeaders.append({"name": key.lower(), "value": value})
if response.get('contentType'):
responseHeaders.append({
"name": 'content-type',
"value": response['contentType']
})
if responseBody and 'content-length' not in responseHeaders:
responseHeaders.append({
"name": 'content-length',
"value": str(len(responseBody))
})
try:
await self._client.send(
'Fetch.fulfillRequest', {
'requestId':
self._interceptionId,
'responseCode':
response.get("status", 200),
'responseHeaders':
responseHeaders,
'body':
base64.b64encode(responseBody).decode('ascii')
if responseBody else None
})
except Exception as e:
debugError(logger, e)
async def respond(self, response: Dict) -> None: # noqa: C901
"""Fulfills request with given response.
To use this, request interception should by enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`. Request
interception is not enabled, raise ``NetworkError``.
``response`` is a dictionary which can have the following fields:
* ``status`` (int): Response status code, defaults to 200.
* ``headers`` (dict): Optional response headers.
* ``contentType`` (str): If set, equals to setting ``Content-Type``
response header.
* ``body`` (str|bytes): Optional response body.
"""
if self._url.startswith('data:'):
return
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
if response.get('body') and isinstance(response['body'], str):
responseBody: Optional[bytes] = response['body'].encode('utf-8')
else:
responseBody = response.get('body')
responseHeaders = {}
if response.get('headers'):
for header in response['headers']:
responseHeaders[header.lower()] = response['headers'][header]
if response.get('contentType'):
responseHeaders['content-type'] = response['contentType']
if responseBody and 'content-length' not in responseHeaders:
responseHeaders['content-length'] = len(responseBody)
statusCode = response.get('status', 200)
statusText = statusTexts.get(statusCode, '')
statusLine = f'HTTP/1.1 {statusCode} {statusText}'
CRLF = '\r\n'
text = statusLine + CRLF
for header in responseHeaders:
text = f'{text}{header}: {responseHeaders[header]}{CRLF}'
text = text + CRLF
responseBuffer = text.encode('utf-8')
if responseBody:
responseBuffer = responseBuffer + responseBody
rawResponse = base64.b64encode(responseBuffer).decode('ascii')
try:
await self._client.send(
'Network.continueInterceptedRequest', {
'interceptionId': self._interceptionId,
'rawResponse': rawResponse,
})
except Exception as e:
debugError(logger, e)
async def dispose(self) -> None:
"""Stop referencing the handle."""
if self._disposed:
return
self._disposed = True
try:
await helper.releaseObject(self._client, self._remoteObject)
except Exception as e:
debugError(logger, e)
async def _getBoxModel(self) -> Optional[Dict]:
try:
result: Optional[Dict] = await self._client.send(
'DOM.getBoxModel',
{'objectId': self._remoteObject.get('objectId')},
)
except NetworkError as e:
debugError(logger, e)
result = None
return result
async def _getBoxModel(self) -> Optional[Dict]:
try:
result: Optional[Dict] = await self._client.send(
'DOM.getBoxModel',
{'objectId': self._remoteObject.get('objectId')},
)
except NetworkError as e:
debugError(logger, e)
result = None
return result
async def killChrome(self) -> None:
'Terminate chromium process.'
logger.info('terminate chrome process...')
if (self.connection and self.connection._connected):
try:
(await self.connection.send('Browser.close'))
(await self.connection.dispose())
except Exception as e:
debugError(logger, e)
if (self._tmp_user_data_dir
and os.path.exists(self._tmp_user_data_dir)):
self.waitForChromeToClose()
self._cleanup_tmp_user_data_dir()
async def respond(self, response: Dict) -> None: # noqa: C901
"""Fulfills request with given response.
To use this, request interception should by enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`. Request
interception is not enabled, raise ``NetworkError``.
``response`` is a dictionary which can have the following fields:
* ``responseCode`` (int): Response status code, defaults to 200.
* ``headers`` (dict): Optional response headers.
* ``body`` (str|bytes): Optional response body.
* ``responsePhrase`` (string): A textual representation of responseCode.
If absent, a standard phrase matching responseCode is used.
"""
if self._url.startswith('data:'):
return
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
if response.get('body') and isinstance(response['body'], str):
responseBody: Optional[bytes] = response['body'].encode('utf-8')
else:
responseBody = response.get('body')
responseHeaders = {}
if response.get('headers'):
for header in response['headers']:
responseHeaders[header.lower()] = response['headers'][header]
if response.get('contentType'):
responseHeaders['content-type'] = response['contentType']
if responseBody and 'content-length' not in responseHeaders:
responseHeaders['content-length'] = len(responseBody)
statusCode = response.get('responseCode', 200)
statusText = statusTexts.get(str(statusCode))
rawResponse = base64.b64encode(responseBody).decode('ascii')
try:
await self._client.send('Fetch.fulfillRequest', {
'requestId': self._interceptionId,
'responseCode': statusCode if statusCode is not None else 200,
'responsePhrase': statusText if statusText is not None else statusTexts['200'],
'responseHeaders': headersArray(responseHeaders),
'body': rawResponse
})
except Exception as e:
debugError(logger, e)
async def _onStyleSheet(self, event: Dict) -> None:
header = event.get('header', {})
# Ignore anonymous scripts
if not header.get('sourceURL'):
return
try:
response = await self._client.send(
'CSS.getStyleSheetText',
{'styleSheetId': header['styleSheetId']})
self._stylesheetURLs[header['styleSheetId']] = header['sourceURL']
self._stylesheetSources[header['styleSheetId']] = response['text']
except Exception as e:
# This might happen if the page has already navigated away.
debugError(logger, e)
async def abort(self, errorCode: str = 'failed') -> None:
"""Abort request.
To use this, request interception should be enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`.
If request interception is not enabled, raise ``NetworkError``.
``errorCode`` is an optional error code string. Defaults to ``failed``,
could be one of the following:
- ``aborted``: An operation was aborted (due to user action).
- ``accessdenied``: Permission to access a resource, other than the
network, was denied.
- ``addressunreachable``: The IP address is unreachable. This usually
means that there is no route to the specified host or network.
- ``blockedbyclient``: The client chose to block the request.
- ``blockedbyresponse``: The request failed because the request was
delivered along with requirements which are not met
('X-Frame-Options' and 'Content-Security-Policy' ancestor check,
for instance).
- ``connectionaborted``: A connection timeout as a result of not
receiving an ACK for data sent.
- ``connectionclosed``: A connection was closed (corresponding to a TCP
FIN).
- ``connectionfailed``: A connection attempt failed.
- ``connectionrefused``: A connection attempt was refused.
- ``connectionreset``: A connection was reset (corresponding to a TCP
RST).
- ``internetdisconnected``: The Internet connection has been lost.
- ``namenotresolved``: The host name could not be resolved.
- ``timedout``: An operation timed out.
- ``failed``: A generic failure occurred.
"""
errorReason = errorReasons[errorCode]
if not errorReason:
raise NetworkError('Unknown error code: {}'.format(errorCode))
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
try:
await self._client.send(
'Network.continueInterceptedRequest',
dict(
interceptionId=self._interceptionId,
errorReason=errorReason,
))
except Exception as e:
debugError(logger, e)
async def killChrome(self) -> None:
"""Terminate chromium process."""
logger.info('terminate chrome process...')
if self.connection and self.connection._connected:
try:
await self.connection.send('Browser.close')
await self.connection.dispose()
except Exception as e:
# ignore errors on browser termination process
debugError(logger, e)
if self._tmp_user_data_dir and os.path.exists(self._tmp_user_data_dir):
# Force kill chrome only when using temporary userDataDir
self.waitForChromeToClose()
self._cleanup_tmp_user_data_dir()
async def _onStyleSheet(self, event: Dict) -> None:
header = event.get('header', {
})
if (not header.get('sourceURL')):
return
try:
response = (await self._client.send('CSS.getStyleSheetText', {
'styleSheetId': header['styleSheetId'],
}))
self._stylesheetURLs[header['styleSheetId']] = header['sourceURL']
self._stylesheetSources[header['styleSheetId']] = response['text']
except Exception as e:
debugError(logger, e)
async def killChrome(self) -> None:
"""Terminate chromium process."""
logger.info('terminate chrome process...')
if self.connection and self.connection._connected:
try:
await self.connection.send('Browser.close')
await self.connection.dispose()
except Exception as e:
# ignore errors on browser termination process
debugError(logger, e)
if self._tmp_user_data_dir and os.path.exists(self._tmp_user_data_dir):
# Force kill chrome only when using temporary userDataDir
self.waitForChromeToClose()
self._cleanup_tmp_user_data_dir()
def __init__(
self,
client: 'CDPSession',
url: str, # noqa: C901
consoleAPICalled: Callable[[str, List[JSHandle]], None],
exceptionThrown: Callable[[Dict], None]) -> None:
super().__init__()
self._client = client
self._url = url
self._loop = client._loop
self._executionContextPromise = self._loop.create_future()
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
return None # type: ignore
def onExecutionContentCreated(event: Dict) -> None:
_execution_contexts: List[ExecutionContext] = []
nonlocal jsHandleFactory
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
executionContext = _execution_contexts[0]
return JSHandle(executionContext, client, remoteObject)
executionContext = ExecutionContext(client, event['context'],
jsHandleFactory)
_execution_contexts.append(executionContext)
self._executionContextCallback(executionContext)
self._client.on('Runtime.executionContextCreated',
onExecutionContentCreated)
try:
# This might fail if the target is closed before we recieve all
# execution contexts.
self._client.send('Runtime.enable', {})
except Exception as e:
debugError(logger, e)
def onConsoleAPICalled(event: Dict) -> None:
args: List[JSHandle] = []
for arg in event.get('args', []):
args.append(jsHandleFactory(arg))
consoleAPICalled(event['type'], args)
self._client.on('Runtime.consoleAPICalled', onConsoleAPICalled)
self._client.on(
'Runtime.exceptionThrown',
lambda exception: exceptionThrown(exception['exceptionDetails']),
)
async def _onScriptParsed(self, event: Dict) -> None:
if (event.get('url') == EVALUATION_SCRIPT_URL):
return
if ((not event.get('url')) and (not self._reportAnonymousScript)):
return
scriptId = event.get('scriptId')
url = event.get('url')
if ((not url) and self._reportAnonymousScript):
url = ''.join(['debugger://VM', '{}'.format(scriptId)])
try:
response = (await self._client.send('Debugger.getScriptSource', {
'scriptId': scriptId,
}))
self._scriptURLs[scriptId] = url
self._scriptSources[scriptId] = response.get('scriptSource')
except Exception as e:
debugError(logger, e)
async def respond(self, response: Dict) -> None:
'Fulfills request with given response.\n\n To use this, request interception should by enabled by\n :meth:`pyppeteer.page.Page.setRequestInterception`. Request\n interception is not enabled, raise ``NetworkError``.\n\n ``response`` is a dictionary which can have the following fields:\n\n * ``status`` (int): Response status code, defaults to 200.\n * ``headers`` (dict): Optional response headers.\n * ``contentType`` (str): If set, equals to setting ``Content-Type``\n response header.\n * ``body`` (str|bytes): Optional response body.\n '
if self._url.startswith('data:'):
return
if (not self._allowInterception):
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
if (response.get('body') and isinstance(response['body'], str)):
responseBody = response['body'].encode('utf-8')
else:
responseBody = response.get('body')
responseHeaders = {
}
if response.get('headers'):
for header in response['headers']:
responseHeaders[header.lower()] = response['headers'][header]
if response.get('contentType'):
responseHeaders['content-type'] = response['contentType']
if (responseBody and ('content-length' not in responseHeaders)):
responseHeaders['content-length'] = len(responseBody)
statusCode = response.get('status', 200)
statusText = statusTexts.get(statusCode, '')
statusLine = ''.join(
['HTTP/1.1 ', '{}'.format(statusCode), ' ', '{}'.format(statusText)])
CRLF = '\r\n'
text = (statusLine + CRLF)
for header in responseHeaders:
text = ''.join(['{}'.format(text), '{}'.format(header), ': ', '{}'.format(
responseHeaders[header]), '{}'.format(CRLF)])
text = (text + CRLF)
responseBuffer = text.encode('utf-8')
if responseBody:
responseBuffer = (responseBuffer + responseBody)
rawResponse = base64.b64encode(responseBuffer).decode('ascii')
try:
(await self._client.send('Network.continueInterceptedRequest', {
'interceptionId': self._interceptionId,
'rawResponse': rawResponse,
}))
except Exception as e:
debugError(logger, e)
def __init__(
self,
client: 'CDPSession',
url: str, # noqa: C901
consoleAPICalled: Callable[[str, List[JSHandle], str], None],
exceptionThrown: Callable[[Dict], None]) -> None:
super().__init__()
self._client = client
self._url = url
self._loop = client._loop
self._executionContextPromise = self._loop.create_future()
jsHandleFactory = lambda remoteObject: remoteObject
def onExecutionContentCreated(event: Dict) -> None:
nonlocal jsHandleFactory
executionContext = ExecutionContext(client, event['context'], None)
jsHandleFactory = functools.partial(JSHandle,
context=executionContext,
client=client)
self._executionContextPromise.set_result(executionContext)
self._client.on('Runtime.executionContextCreated',
onExecutionContentCreated)
try:
# This might fail if the target is closed before we receive all
# execution contexts.
self._client.send('Runtime.enable', {})
except Exception as e:
debugError(logger, e)
def onConsoleAPICalled(event: Dict) -> None:
consoleAPICalled(
event['type'],
list(
map(lambda x: jsHandleFactory(remoteObject=x),
event.get('args', []))), event.get('stackTrace'))
self._client.on('Runtime.consoleAPICalled', onConsoleAPICalled)
self._client.on(
'Runtime.exceptionThrown',
lambda exception: exceptionThrown(exception['exceptionDetails']),
)
async def _recv_loop(self) -> None:
async with self._ws as connection:
self._connected = True
self.connection = connection
while self._connected:
try:
resp = await self.connection.recv()
if resp:
await self._on_message(resp)
except (websockets.ConnectionClosed,
ConnectionResetError) as e:
logger.info('connection closed')
break
except Exception as e:
debugError(logger, e)
break
await asyncio.sleep(0)
if self._connected:
self._loop.create_task(self.dispose())
def __init__(self, client: 'CDPSession', url: str, # noqa: C901
consoleAPICalled: Callable[[str, List[JSHandle]], None],
exceptionThrown: Callable[[Dict], None]
) -> None:
super().__init__()
self._client = client
self._url = url
self._loop = client._loop
self._executionContextPromise = self._loop.create_future()
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
return None # type: ignore
def onExecutionContentCreated(event: Dict) -> None:
nonlocal jsHandleFactory
def jsHandleFactory(remoteObject: Dict) -> JSHandle:
return JSHandle(executionContext, client, remoteObject)
executionContext = ExecutionContext(
client, event['context'], jsHandleFactory)
self._executionContextCallback(executionContext)
self._client.on('Runtime.executionContextCreated',
onExecutionContentCreated)
try:
# This might fail if the target is closed before we receive all
# execution contexts.
self._client.send('Runtime.enable', {})
except Exception as e:
debugError(logger, e)
def onConsoleAPICalled(event: Dict) -> None:
args: List[JSHandle] = []
for arg in event.get('args', []):
args.append(jsHandleFactory(arg))
consoleAPICalled(event['type'], args)
self._client.on('Runtime.consoleAPICalled', onConsoleAPICalled)
self._client.on(
'Runtime.exceptionThrown',
lambda exception: exceptionThrown(exception['exceptionDetails']),
)
async def _onScriptParsed(self, event: Dict) -> None:
# Ignore pyppeteer-injected scripts
if event.get('url') == EVALUATION_SCRIPT_URL:
return
# Ignore other anonymous scripts unless the reportAnonymousScript
# option is True
if not event.get('url') and not self._reportAnonymousScript:
return
scriptId = event.get('scriptId')
url = event.get('url')
if not url and self._reportAnonymousScript:
url = f'debugger://VM{scriptId}'
try:
response = await self._client.send('Debugger.getScriptSource',
{'scriptId': scriptId})
self._scriptURLs[scriptId] = url
self._scriptSources[scriptId] = response.get('scriptSource')
except Exception as e:
# This might happen if the page has already navigated away.
debugError(logger, e)
async def continue_(self, overrides: Dict = None) -> None:
"""Continue request with optional request overrides.
To use this method, request interception should be enabled by
:meth:`pyppeteer.page.Page.setRequestInterception`. If request
interception is not enabled, raise ``NetworkError``.
``overrides`` can have the following fields:
* ``url`` (str): If set, the request url will be changed.
* ``method`` (str): If set, change the request method (e.g. ``GET``).
* ``postData`` (str): If set, change the post data or request.
* ``headers`` (dict): If set, change the request HTTP header.
"""
if overrides is None:
overrides = {}
if not self._allowInterception:
raise NetworkError('Request interception is not enabled.')
if self._interceptionHandled:
raise NetworkError('Request is already handled.')
self._interceptionHandled = True
url = overrides.get('url')
method = overrides.get('method')
postData = overrides.get('postData')
headers = overrides.get('headers')
try:
await self._client.send('Fetch.continueRequest',
{'requestId': self._interceptionId,
'url': url,
'method': method,
'postData': postData,
'headers': headersArray(headers) if headers else None
}
)
except Exception as e:
debugError(logger, e)
def test_debug_enable_disable(self):
pyppeteer.DEBUG = True
with self.assertLogs('pyppeteer.test', logging.ERROR):
debugError(self.logger, 'test')
pyppeteer.DEBUG = False
with self.assertLogs('pyppeteer.test', logging.DEBUG):
debugError(self.logger, 'test')
with self.assertRaises(AssertionError):
with self.assertLogs('pyppeteer.test', logging.INFO):
debugError(self.logger, 'test')
def test_debug_logger(self):
with self.assertRaises(AssertionError):
with self.assertLogs('pyppeteer', logging.DEBUG):
debugError(logging.getLogger('test'), 'test message')
def test_debug_enabled(self):
pyppeteer.DEBUG = True
with self.assertLogs('pyppeteer.test', logging.ERROR):
debugError(self.logger, 'test')
def test_debug_default(self):
with self.assertLogs('pyppeteer.test', logging.DEBUG):
debugError(self.logger, 'test')
with self.assertRaises(AssertionError):
with self.assertLogs('pyppeteer', logging.INFO):
debugError(self.logger, 'test')
async def _send(self, method: str, msg: dict) -> None:
try:
await self._client.send(method, msg)
except Exception as e:
debugError(logger, e)