def _create_request(self, flow, response=None):
request = Request(method=flow.request.method,
url=flow.request.url,
headers=[(k, v)
for k, v in flow.request.headers.items()],
body=flow.request.raw_content)
request.response = response
return request
def default_interceptor(request: selenium_request.Request):
"""
Inner function that acts like a proxy, it removes any requests we don't want.
@param request: The current request
@type request: selenium_request.Request
@return: None
"""
# Block PNG, JPEG and GIF images.
if request.path.endswith(('.png', '.jpg', '.gif')):
request.abort() # Abort the unwanted request.
def interceptor(request: Request) -> None:
if request.path.endswith((".png", ".jpg", ".gif")):
request.abort()
return
if (
"edmunds.com" not in request.host
and "edmunds-media.com" not in request.host
):
request.abort()
return
if "activate" in request.path or "certified-program" in request.path:
request.abort()
return
request.headers["User-Agent"] = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/99.0.4844.82 Safari/537.36 Vivaldi/4.3"
)
if (ffkey := "fetchSuggestedFacets") in request.params.keys():
new_params = request.params
new_params[ffkey] = "false"
request.params = new_params
def _create_request(self, flow, response=None):
request = Request(
method=flow.request.method,
url=flow.request.url,
headers=[(k, v) for k, v in flow.request.headers.items()],
body=flow.request.raw_content
)
# For websocket requests, the scheme of the request is overwritten with https
# in the initial CONNECT request so this hack explicitly sets the scheme back to wss.
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
request.url = request.url.replace('https', 'wss', 1)
request.response = response
return request
def _create_request(self, body=None):
request = Request(
method='GET',
url=
'http://www.example.com/some/path/?foo=bar&hello=world&foo=baz&other=',
headers=
[('Accept', '*/*'), ('Host', 'www.example.com'),
('User-Agent',
'Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0'
)],
body=body)
return request
def save_request(self, request: Request) -> None:
"""Save a request to storage.
Args:
request: The request to save.
"""
request.id = str(uuid.uuid4())
with self._lock:
if self._maxsize > 0:
while len(self._requests) >= self._maxsize:
self._requests.popitem(last=False)
self._requests[request.id] = {
'request': request,
}
def save_request(self, request: Request) -> None:
"""Save a request to storage.
Args:
request: The request to save.
"""
request_id = str(uuid.uuid4())
request_dir = self._get_request_dir(request_id)
os.mkdir(request_dir)
request.id = request_id
self._save(request, request_dir, 'request')
with self._lock:
self._index.append(
_IndexedRequest(id=request_id,
url=request.url,
has_response=False))
def _create_request(self,
url='http://www.example.com/test/path/',
body=b''):
headers = [('Host', 'www.example.com'), ('Accept', '*/*')]
return Request(method='GET', url=url, headers=headers, body=body)
def _intercept_requests(self, request: Request):
# Intercept Network Requests - E.g. Block Ads/Malware
if self.should_block_request(url=request.url):
request.abort()