def get_resource(link):
# type: (Text) -> VGResource
if link.startswith("data:"):
# resource already in blob
return VGResource.EMPTY(link)
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
def test_vgresource_with_function_that_raises_exception_should_not_break():
def _raise():
raise Exception
VGResource.from_blob(
{
"url": "https://someurl.com",
"type": "application/empty-response",
"content": b"",
},
_raise,
)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content):
urls_from_css, urls_from_svg = [], []
if content_type.startswith("text/css"):
urls_from_css = parsers.get_urls_from_css_resource(content)
if content_type.startswith("image/svg"):
urls_from_svg = parsers.get_urls_from_svg_resource(content)
for discovered_url in urls_from_css + urls_from_svg:
target_url = _apply_base_url(discovered_url, base_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
if link.startswith("data:"):
# resource already in blob
return VGResource.EMPTY(link)
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls + discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url, val in iteritems(self.resource_cache):
if val is None:
val = VGResource.EMPTY(r_url)
self.request_resources[r_url] = val
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def process_all(self):
for r_url, val in iteritems(self):
if val is None:
logger.debug("No response for {}".format(r_url))
val = VGResource.EMPTY(r_url)
self[r_url] = val
return self
def test_render_request_serialize(browser_type):
request_resources = {"url": VGResource.EMPTY("some-url.com")}
dom_url = "dom-url.com"
r_info = RenderInfo(
width=500,
height=600,
size_mode="full-page",
selector=None,
region=None,
emulation_info=None,
)
dom = RGridDom(
url=dom_url,
dom_nodes=[{}],
resources=request_resources,
)
requests = [
RenderRequest(
webhook="some-webhook.com",
agent_id="my-agent-id",
url=dom_url,
dom=dom,
resources=request_resources,
render_info=r_info,
browser_name=browser_type,
platform="linux",
script_hooks=dict(),
selectors_to_find_regions_for=[],
send_dom=False,
)
]
RESULT_PATTERN = '[{"agentId": "my-agent-id", "browser": {"name": "%s", "platform": "linux"}, "dom": {"hash": "a67486a8bc9ba45f878e5b0d8ff9bc68ec6ed9db0382709751327d1793898e16", "hashFormat": "sha256"}, "renderId": null, "renderInfo": {"height": 600, "sizeMode": "full-page", "width": 500}, "resources": {"url": {"contentType": "application/empty-response", "hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "hashFormat": "sha256"}}, "scriptHooks": {}, "selectorsToFindRegionsFor": [], "sendDom": false, "url": "dom-url.com", "webhook": "some-webhook.com"}]'
assert RESULT_PATTERN % (browser_type.value) == json_utils.to_json(
requests)
def get_resource(link):
# type: (Text) -> VGResource
logger.debug("get_resource({0}) call".format(link))
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
logger.debug("parse_frame_dom_resources() call")
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content, resource_url):
logger.debug("handle_resources({0}) call".format(content_type))
urls_from_css, urls_from_svg = [], []
if content_type.startswith("text/css"):
urls_from_css = parsers.get_urls_from_css_resource(content)
if content_type.startswith("image/svg"):
urls_from_svg = parsers.get_urls_from_svg_resource(content)
for discovered_url in urls_from_css + urls_from_svg:
target_url = _apply_base_url(discovered_url, base_url,
resource_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
logger.debug("get_resource({0}) call".format(link))
if link.startswith("data:"):
# resource already in blob
return VGResource.EMPTY(link)
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls + discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
# some discovered urls becomes available only after resources processed
for r_url in set(discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
self.request_resources.update(self.resource_cache)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
all_blobs = data.get("blobs", [])
frames = data.get("frames", [])
logger.debug("""
parse_frame_dom_resources() call
base_url: {base_url}
count blobs: {blobs_num}
count resource urls: {resource_urls_num}
count frames: {frames_num}
""".format(
base_url=base_url,
blobs_num=len(all_blobs),
resource_urls_num=len(resource_urls),
frames_num=len(frames),
))
def find_child_resource_urls(content_type, content, resource_url):
# type: (Optional[Text], bytes, Text) -> NoReturn
logger.debug("find_child_resource_urls({0}, {1}) call".format(
content_type, resource_url))
if not content_type:
logger.debug(
"content_type is empty. Skip handling of resources")
return []
return [
apply_base_url(url, base_url, resource_url)
for url in collect_urls_from_(content_type, content)
]
frame_request_resources = {}
for f_data in frames:
f_data["url"] = apply_base_url(f_data["url"], base_url)
frame_request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
urls_to_fetch = set(resource_urls)
for blob in all_blobs:
resource = VGResource.from_blob(blob, find_child_resource_urls)
if resource.url.rstrip("#") == base_url:
continue
frame_request_resources[resource.url] = resource
urls_to_fetch |= set(resource.child_resource_urls)
resources_and_their_children = fetch_resources_recursively(
urls_to_fetch,
self.server_connector,
self.resource_cache,
find_child_resource_urls,
)
frame_request_resources.update(resources_and_their_children)
self.full_request_resources.update(frame_request_resources)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=frame_request_resources)
def test_vg_resource_big_content_should_be_cutted_in_vg_resource():
chars = (b"a", b"b", b"c", b"2")
resource = VGResource(
"https://test.url",
"content-type/test",
b"".join(chars[randint(0, 3)] for _ in range(VGResource.MAX_RESOURCE_SIZE + 5)),
)
assert len(resource.content) == VGResource.MAX_RESOURCE_SIZE
def process_all(self):
with self.lock:
for r_url in self:
val = self.get(r_url)
if val is None:
logger.debug("No response for {}".format(r_url))
val = VGResource.EMPTY(r_url)
self[r_url] = val
return self
def prepare_data_for_rg(self, data):
# type: (Dict) -> RenderRequest
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
for blob in blobs:
resource = VGResource.from_blob(blob)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
if resource.content_type == "text/css":
urls_from_css = _get_urls_from_css_resource(resource)
resource_urls.extend(urls_from_css)
def get_resource(link):
# type: (Text) -> VGResource
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link, response)
for r_url in set(resource_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.request_resources.update(self.resource_cache)
r_info = RenderInfo(
width=self.running_test.browser_info.width,
height=self.running_test.browser_info.height,
size_mode=self.size_mode,
region=self.region_to_check,
emulation_info=self.running_test.browser_info.emulation_info,
)
dom = RGridDom(
url=base_url, dom_nodes=data["cdt"], resources=self.request_resources
)
return RenderRequest(
webhook=self.rendering_info.results_url,
agent_id=self.agent_id,
url=base_url,
dom=dom,
resources=self.request_resources,
render_info=r_info,
browser_name=self.running_test.browser_info.browser_type,
platform=self.running_test.browser_info.platform,
script_hooks={},
selectors_to_find_regions_for=self.region_selectors,
send_dom=self.running_test.configuration.send_dom,
)
def parse_frame_dom_resources(self, data):
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
blobs = data.get("blobs", [])
frames = data.get("frames", [])
discovered_resources_urls = []
def handle_resources(content_type, content):
if content_type == "text/css":
urls_from_css = css_parser.get_urls_from_css_resource(content)
for discovered_url in urls_from_css:
target_url = _apply_base_url(discovered_url, base_url)
with self.discovered_resources_lock:
discovered_resources_urls.append(target_url)
def get_resource(link):
# type: (Text) -> VGResource
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = _apply_base_url(f_data["url"], base_url)
self.request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
self.all_blobs.append(resource)
self.request_resources[resource.url] = resource
for r_url in set(resource_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url in discovered_resources_urls:
self.resource_cache.fetch_and_store(r_url, get_resource)
self.request_resources.update(self.resource_cache)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=self.request_resources)
def test_render_request_serialize(browser_type):
request_resources = {
"url": VGResource(
"some-url.com", content_type="application/png", content=b"some-content"
)
}
dom_url = "dom-url.com"
r_info = RenderInfo(
width=500,
height=600,
size_mode="full-page",
selector=None,
region=None,
emulation_info=None,
)
dom = RGridDom(url=dom_url, dom_nodes=[{}], resources=request_resources)
requests = [
RenderRequest(
webhook="some-webhook.com",
agent_id="my-agent-id",
stitching_service="https://some.stitchingserviceuri.com",
url=dom_url,
dom=dom,
resources=request_resources,
render_info=r_info,
browser_name=browser_type.value,
platform_name="linux",
script_hooks=dict(),
selectors_to_find_regions_for=[],
send_dom=False,
)
]
test_results_data = get_resource("unit/renderResult.json").decode("utf-8")
test_results_data %= browser_type.value
assert json.loads(test_results_data.replace("\n", "")) == json.loads(
json_utils.to_json(requests)
)
def get_resource(link):
# type: (Text) -> VGResource
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link, response)
def parse_frame_dom_resources(self, data): # noqa
# type: (Dict) -> RGridDom
base_url = data["url"]
resource_urls = data.get("resourceUrls", [])
all_blobs = data.get("blobs", [])
frames = data.get("frames", [])
logger.debug("""
parse_frame_dom_resources() call
base_url: {base_url}
count blobs: {blobs_num}
count resource urls: {resource_urls_num}
count frames: {frames_num}
""".format(
base_url=base_url,
blobs_num=len(all_blobs),
resource_urls_num=len(resource_urls),
frames_num=len(frames),
))
frame_request_resources = {}
discovered_resources_urls = set()
def handle_resources(content_type, content, resource_url):
# type: (Optional[Text], bytes, Text) -> NoReturn
logger.debug("handle_resources({0}, {1}) call".format(
content_type, resource_url))
if not content_type:
logger.debug(
"content_type is empty. Skip handling of resources")
return
for url in collect_urls_from_(content_type, content):
target_url = apply_base_url(url, base_url, resource_url)
discovered_resources_urls.add(target_url)
def get_resource(link):
# type: (Text) -> VGResource
logger.debug("get_resource({0}) call".format(link))
response = self.eyes_connector.download_resource(link)
return VGResource.from_response(link,
response,
on_created=handle_resources)
for f_data in frames:
f_data["url"] = apply_base_url(f_data["url"], base_url)
frame_request_resources[f_data[
"url"]] = self.parse_frame_dom_resources(f_data).resource
for blob in all_blobs:
resource = VGResource.from_blob(blob, on_created=handle_resources)
if resource.url.rstrip("#") == base_url:
continue
frame_request_resources[resource.url] = resource
for r_url in set(resource_urls).union(discovered_resources_urls):
self.resource_cache.fetch_and_store(r_url, get_resource)
self.resource_cache.process_all()
# some discovered urls becomes available only after resources processed
for r_url in discovered_resources_urls:
self.resource_cache.fetch_and_store(r_url, get_resource)
for r_url in set(resource_urls).union(discovered_resources_urls):
val = self.resource_cache[r_url]
if val is None:
logger.debug("No response for {}".format(r_url))
continue
frame_request_resources[r_url] = val
self.full_request_resources.update(frame_request_resources)
return RGridDom(url=base_url,
dom_nodes=data["cdt"],
resources=frame_request_resources)
def get_resource(link):
logger.debug("get_resource({0}) call".format(link))
matching_cookies = [c for c in cookies if is_cookie_for_url(c, link)]
response = eyes_connector.download_resource(link, matching_cookies)
return VGResource.from_response(link, response,
find_child_resource_urls)
def get_resource(link):
logger.debug("get_resource({0}) call".format(link))
response = eyes_connector.download_resource(link)
return VGResource.from_response(link, response, find_child_resource_urls)