def test_vgresource_with_function_that_raises_exception_should_not_break():
def _raise():
raise Exception
VGResource.from_blob(
{
"url": "https://someurl.com",
"type": "application/empty-response",
"content": b"",
},
_raise,
)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
logger.debug("parse_frame_dom_resources() call")
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content, resource_url):
logger.debug("handle_resources({0}) call".format(content_type))
urls_from_css, urls_from_svg = [], []
if content_type.startswith("text/css"):
urls_from_css = parsers.get_urls_from_css_resource(content)
if content_type.startswith("image/svg"):
urls_from_svg = parsers.get_urls_from_svg_resource(content)
for discovered_url in urls_from_css + urls_from_svg:
target_url = _apply_base_url(discovered_url, base_url,
resource_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
logger.debug("get_resource({0}) call".format(link))
if link.startswith("data:"):
# resource already in blob
return VGResource.EMPTY(link)
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls + discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
# some discovered urls becomes available only after resources processed
for r_url in set(discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
self.request_resources.update(self.resource_cache)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
all_blobs = data.get("blobs", [])
frames = data.get("frames", [])
logger.debug("""
parse_frame_dom_resources() call
base_url: {base_url}
count blobs: {blobs_num}
count resource urls: {resource_urls_num}
count frames: {frames_num}
""".format(
base_url=base_url,
blobs_num=len(all_blobs),
resource_urls_num=len(resource_urls),
frames_num=len(frames),
))
def find_child_resource_urls(content_type, content, resource_url):
# type: (Optional[Text], bytes, Text) -> NoReturn
logger.debug("find_child_resource_urls({0}, {1}) call".format(
content_type, resource_url))
if not content_type:
logger.debug(
"content_type is empty. Skip handling of resources")
return []
return [
apply_base_url(url, base_url, resource_url)
for url in collect_urls_from_(content_type, content)
]
frame_request_resources = {}
for f_data in frames:
f_data["url"] = apply_base_url(f_data["url"], base_url)
frame_request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
urls_to_fetch = set(resource_urls)
for blob in all_blobs:
resource = VGResource.from_blob(blob, find_child_resource_urls)
if resource.url.rstrip("#") == base_url:
continue
frame_request_resources[resource.url] = resource
urls_to_fetch |= set(resource.child_resource_urls)
resources_and_their_children = fetch_resources_recursively(
urls_to_fetch,
self.server_connector,
self.resource_cache,
find_child_resource_urls,
)
frame_request_resources.update(resources_and_their_children)
self.full_request_resources.update(frame_request_resources)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=frame_request_resources)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content):
urls_from_css, urls_from_svg = [], []
if content_type.startswith("text/css"):
urls_from_css = parsers.get_urls_from_css_resource(content)
if content_type.startswith("image/svg"):
urls_from_svg = parsers.get_urls_from_svg_resource(content)
for discovered_url in urls_from_css + urls_from_svg:
target_url = _apply_base_url(discovered_url, base_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
if link.startswith("data:"):
# resource already in blob
return VGResource.EMPTY(link)
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls + discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url, val in iteritems(self.resource_cache):
if val is None:
val = VGResource.EMPTY(r_url)
self.request_resources[r_url] = val
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def prepare_data_for_rg(self, data):
# type: (Dict) -> RenderRequest
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
for blob in blobs:
resource = VGResource.from_blob(blob)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
if resource.content_type == "text/css":
urls_from_css = _get_urls_from_css_resource(resource)
resource_urls.extend(urls_from_css)
def get_resource(link):
# type: (Text) -> VGResource
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link, response)
for r_url in set(resource_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.request_resources.update(self.resource_cache)
r_info = RenderInfo(
width=self.running_test.browser_info.width,
height=self.running_test.browser_info.height,
size_mode=self.size_mode,
region=self.region_to_check,
emulation_info=self.running_test.browser_info.emulation_info,
)
dom = RGridDom(
url=base_url, dom_nodes=data["cdt"], resources=self.request_resources
)
return RenderRequest(
webhook=self.rendering_info.results_url,
agent_id=self.agent_id,
url=base_url,
dom=dom,
resources=self.request_resources,
render_info=r_info,
browser_name=self.running_test.browser_info.browser_type,
platform=self.running_test.browser_info.platform,
script_hooks={},
selectors_to_find_regions_for=self.region_selectors,
send_dom=self.running_test.configuration.send_dom,
)
def parse_frame_dom_resources(self, data):
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content):
if content_type == "text/css":
urls_from_css = css_parser.get_urls_from_css_resource(content)
for discovered_url in urls_from_css:
target_url = _apply_base_url(discovered_url, base_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url in discovered_resources_urls:
self.resource_cache.fetch_and_store(r_url, get_resource)
self.request_resources.update(self.resource_cache)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
all_blobs = data.get("blobs", [])
frames = data.get("frames", [])
logger.debug("""
parse_frame_dom_resources() call
base_url: {base_url}
count blobs: {blobs_num}
count resource urls: {resource_urls_num}
count frames: {frames_num}
""".format(
base_url=base_url,
blobs_num=len(all_blobs),
resource_urls_num=len(resource_urls),
frames_num=len(frames),
))
frame_request_resources = {}
discovered_resources_urls = set()
def handle_resources(content_type, content, resource_url):
# type: (Optional[Text], bytes, Text) -> NoReturn
logger.debug("handle_resources({0}, {1}) call".format(
content_type, resource_url))
if not content_type:
logger.debug(
"content_type is empty. Skip handling of resources")
return
for url in collect_urls_from_(content_type, content):
target_url = apply_base_url(url, base_url, resource_url)
discovered_resources_urls.add(target_url)
def get_resource(link):
# type: (Text) -> VGResource
logger.debug("get_resource({0}) call".format(link))
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = apply_base_url(f_data["url"], base_url)
frame_request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in all_blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
frame_request_resources[resource.url] = resource
for r_url in set(resource_urls).union(discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
# some discovered urls becomes available only after resources processed
for r_url in discovered_resources_urls:
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url in set(resource_urls).union(discovered_resources_urls):
val = self.resource_cache[r_url]
if val is None:
logger.debug("No response for {}".format(r_url))
continue
frame_request_resources[r_url] = val
self.full_request_resources.update(frame_request_resources)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=frame_request_resources)