def _process_future(self, val):
if isinstance(val, Future):
try:
return val.result()
except Exception as e:
logger.exception(e)
return None
return val
def send_logs(self, *client_events):
# type: (*ClientEvent) -> None
# try once
try:
events = json_utils.to_json(LogSessionsClientEvents(client_events))
self._com.request("post", self.API_SESSIONS_LOG, data=events)
except Exception:
logger.exception("send_logs failed")
def perform(self): # noqa
# type: () -> List[RenderStatusResults]
def get_and_put_resource(url, running_render):
# type: (str, RunningRender) -> VGResource
logger.debug(
"get_and_put_resource({0}, render_id={1}) call".format(
url, running_render.render_id))
resource = self.request_resources.get(url)
self.eyes_connector.render_put_resource(running_render, resource)
return resource
requests = self.prepare_data_for_rg(self.script)
fetch_fails = 0
render_requests = None
already_force_putted = False
while True:
try:
self.put_cache.process_all()
render_requests = self.eyes_connector.render(*requests)
except Exception as e:
logger.exception(e)
fetch_fails += 1
datetime_utils.sleep(
1500, msg="/render throws exception... sleeping for 1.5s")
if not render_requests:
logger.error("running_renders is null")
continue
need_more_dom = need_more_resources = False
for i, running_render in enumerate(render_requests):
requests[i].render_id = running_render.render_id
need_more_dom = running_render.need_more_dom
need_more_resources = (running_render.render_status ==
RenderStatus.NEED_MORE_RESOURCE)
get_and_put_resource_wtih_render = partial(
get_and_put_resource, running_render=running_render)
dom_resource = requests[i].dom.resource
if self.is_force_put_needed and not already_force_putted:
for url in self.request_resources:
self.put_cache.fetch_and_store(
url, get_and_put_resource_wtih_render)
already_force_putted = True
if need_more_resources:
for url in running_render.need_more_resources:
self.put_cache.fetch_and_store(
url, get_and_put_resource_wtih_render)
if need_more_dom:
self.eyes_connector.render_put_resource(
running_render, dom_resource)
still_running = (need_more_resources or need_more_dom
or fetch_fails > self.MAX_FAILS_COUNT)
if not still_running:
break
return self.poll_render_status(requests)
def _url_from_tags(tags):
for tag in tags:
if tag.type == "url":
try:
url = urlparse(tag.value)
if url.scheme in ["http", "https"]:
yield url.geturl()
except Exception as e:
logger.exception(e)
def perform(self):
# type: () -> RenderStatusResults
def get_and_put_resource(url):
# type: (str) -> VGResource
resource = self.request_resources.get(url)
self.eyes_connector.render_put_resource(running_render, resource)
return resource
requests = []
rq = self.prepare_data_for_rg(self.script)
requests.append(rq)
fetch_fails = 0
render_requests = None
while True:
try:
render_requests = self.eyes_connector.render(*requests)
except Exception as e:
logger.exception(e)
fetch_fails += 1
time.sleep(1.5)
if not render_requests:
continue
running_render = render_requests[0]
rq.render_id = running_render.render_id
need_more_dom = running_render.need_more_dom
need_more_resources = (
running_render.render_status == RenderStatus.NEED_MORE_RESOURCE
)
still_running = (
need_more_resources
or need_more_dom
or fetch_fails > self.MAX_FAILS_COUNT
)
dom_resource = rq.dom.resource
if need_more_resources:
for url in running_render.need_more_resources:
self.put_cache.fetch_and_store(url, get_and_put_resource)
if need_more_dom:
self.eyes_connector.render_put_resource(running_render, dom_resource)
if not still_running:
break
statuses = self.poll_render_status(rq)
if statuses and statuses[0].status == RenderStatus.ERROR:
raise EyesError(
"Render failed for {} with the message: {}".format(
statuses[0].status, statuses[0].error
)
)
return statuses[0]
def __attrs_post_init__(self):
self.hash = general_utils.get_sha256_hash(self.content)
if callable(self._handle_func):
try:
self._handle_func()
except Exception:
logger.exception(
"Exception has been appeared during processing"
" of VGResource({})".format(self.url),
)
def _check_element(self, name, check_settings):
element = self._target_element # type: EyesWebElement
self._region_to_check = None
scroll_root_element = eyes_selenium_utils.current_frame_scroll_root_element(
self.driver)
pos_provider = self._create_position_provider(scroll_root_element)
result = None
with pos_provider:
self._ensure_element_visible(element)
pl = element.location
try:
self._check_frame_or_element = True
display_style = element.get_computed_style("display")
if self.configuration.hide_scrollbars:
# FIXME bug if trying to hide
element.hide_scrollbars()
size_and_borders = element.size_and_borders
border_widths = size_and_borders.borders
element_size = size_and_borders.size
if display_style != "inline" and (
element_size["height"] <=
self._effective_viewport["height"]
and element_size["width"] <=
self._effective_viewport["width"]):
self._element_position_provider = ElementPositionProvider(
self.driver, element)
else:
self._element_position_provider = None
element_region = Region(
pl["x"] + border_widths["left"],
pl["y"] + border_widths["top"],
element_size["width"],
element_size["height"],
coordinates_type=CoordinatesType.SCREENSHOT_AS_IS,
)
self._region_to_check = element_region
if not self._effective_viewport.is_size_empty:
self._region_to_check.intersect(self._effective_viewport)
result = self._check_window_base(NULL_REGION_PROVIDER, name,
False, check_settings)
except Exception as e:
logger.exception(e)
raise e
finally:
if self.configuration.hide_scrollbars:
element.return_to_original_overflow()
self._check_frame_or_element = False
self._region_to_check = None
self._element_position_provider = None
return result
def _process_future(self, val):
if isinstance(val, Future):
try:
val = val.result()
except HTTPError as e:
logger.error("Resource haven't been downloaded.")
logger.exception(e)
except Exception as e:
self.executor.shutdown()
raise e
return val
def _process_future(self, url, val):
if isinstance(val, Future):
try:
val = val.result()
except Exception:
logger.exception(
"We got an exception during processing URL: {}".format(url)
)
val = None
finally:
self.cache_map[url] = val
return val
def set_overflow(driver, overflow, root_element):
# type: (EyesWebDriver, Text, AnyWebElement) -> Optional[Text]
root_element = get_underlying_webelement(root_element)
with timeout(0.1):
try:
return driver.execute_script(
_JS_SET_OVERFLOW % (overflow, overflow), root_element)
except WebDriverException as e:
logger.warning("Couldn't sent overflow {} to element {}".format(
overflow, root_element))
logger.exception(e)
return None
def check(self, check_settings):
# type: (SeleniumCheckSettings) -> None
argument_guard.is_a(check_settings, CheckSettings)
name = check_settings.values.name
logger.debug("VisualGridEyes.check(%s, %s)" % (name, check_settings))
self._try_set_target_selector(check_settings)
self.configure.send_dom = check_settings.values.send_dom
check_settings = self._update_check_settings(check_settings)
logger.info("check('{}', check_settings) - begin".format(name))
region_xpaths = self.get_region_xpaths(check_settings)
logger.info("region_xpaths: {}".format(region_xpaths))
dont_fetch_resources = self._effective_disable_browser_fetching(
self.configure, check_settings)
running_tests = [
test for test in self.test_list
if self._test_uuid == test.test_uuid
]
try:
script_result = self.get_script_result(dont_fetch_resources)
logger.debug("Cdt length: {}".format(len(script_result["cdt"])))
logger.debug("Blobs urls: {}".format(
[b["url"] for b in script_result["blobs"]]))
logger.debug("Resources urls: {}".format(
script_result["resourceUrls"]))
source = eyes_selenium_utils.get_check_source(self.driver)
for test in running_tests:
test.check(
tag=name,
check_settings=check_settings,
script_result=script_result,
visual_grid_manager=self.vg_manager,
region_selectors=region_xpaths,
region_to_check=check_settings.values.target_region,
script_hooks=check_settings.values.script_hooks,
source=source,
)
if test.state == "new":
test.becomes_not_rendered()
except Exception as e:
logger.exception(e)
for test in running_tests:
if test.state != "tested":
# already aborted or closed
test.abort()
test.becomes_tested()
logger.info("added check tasks {}".format(check_settings))
def paste_image(base_image, part_image, position):
# type: (Image.Image, Image.Image, Point) -> Image.Image
argument_guard.is_a(base_image, Image.Image)
argument_guard.is_a(part_image, Image.Image)
argument_guard.is_a(position, Point)
try:
base_image.paste(part_image, box=(position.x, position.y))
except TypeError as e:
logger.exception(e)
logger.debug("Convert position values to integer")
base_image.paste(part_image,
box=(round_converter(position.x),
round_converter(position.y)))
return base_image
def __attrs_post_init__(self):
if len(self.content) > self.MAX_RESOURCE_SIZE:
logger.debug(
"The content of {} is bigger then supported max size. "
"Trimming to {} bytes".format(self.url,
self.MAX_RESOURCE_SIZE))
self.content = self.content[:self.MAX_RESOURCE_SIZE]
self.hash = general_utils.get_sha256_hash(self.content)
if not self.error_status_code and callable(
self._get_child_resource_urls_func):
try:
self.child_resource_urls = self._get_child_resource_urls_func(
self.content_type, self.content, self.url)
except Exception:
logger.exception(
"Exception has been appeared during processing"
" of VGResource({})".format(self.url), )
def perform(self):
# type: () -> RenderStatusResults
def get_and_put_resource(url):
# type: (str) -> VGResource
resource = self.request_resources.get(url)
self.eyes_connector.render_put_resource(running_render, resource)
return resource
requests = self.prepare_data_for_rg(self.script)
fetch_fails = 0
render_requests = None
while True:
try:
render_requests = self.eyes_connector.render(*requests)
except Exception as e:
logger.exception(e)
fetch_fails += 1
datetime_utils.sleep(1500)
if not render_requests:
continue
need_more_dom = need_more_resources = False
for i, running_render in enumerate(render_requests):
requests[i].render_id = running_render.render_id
need_more_dom = running_render.need_more_dom
need_more_resources = (running_render.render_status ==
RenderStatus.NEED_MORE_RESOURCE)
dom_resource = requests[i].dom.resource
if need_more_resources:
for url in running_render.need_more_resources:
self.put_cache.fetch_and_store(url,
get_and_put_resource)
if need_more_dom:
self.eyes_connector.render_put_resource(
running_render, dom_resource)
still_running = (need_more_resources or need_more_dom
or fetch_fails > self.MAX_FAILS_COUNT)
if not still_running:
break
return self.poll_render_status(requests)
def __call__(self):
# type: () -> Optional[TestResults]
logger.debug("%s called %s" % (self.__class__.__name__, self.name))
res = None
try:
if callable(self.func_to_run):
res = self.func_to_run()
if callable(self.callback):
self.callback(res)
except Exception as e:
logger.error("Failed to execute task! \n\t %s" % self.name)
logger.exception(e)
if callable(self.error_callback):
self.error_callback(e)
finally:
if callable(self.complete_callback):
self.complete_callback()
return res
def _set_viewport_size(self):
viewport_size = self.configure.viewport_size
if viewport_size is None:
for render_bi in self.configure.browsers_info:
if not render_bi.emulation_info:
viewport_size = RectangleSize(render_bi.width, render_bi.height)
break
if viewport_size is None:
viewport_size = self._get_viewport_size()
self.configure.viewport_size = viewport_size
try:
eyes_selenium_utils.set_viewport_size(self.driver, viewport_size)
except Exception as e:
logger.exception(e)
raise EyesError(str(e))
def _try_upload_data(self, bytes_data, content_type, media_type):
# type: (bytes, Text, Text) -> Optional[Text]
argument_guard.not_none(bytes_data)
rendering_info = self.render_info()
if rendering_info and rendering_info.results_url:
try:
target_url = rendering_info.results_url
guid = uuid.uuid4()
target_url = target_url.replace("__random__", str(guid))
logger.debug("Uploading {} to {}".format(media_type, target_url))
if self._upload_data(
bytes_data, rendering_info, target_url, content_type, media_type
):
return target_url
except Exception as e:
logger.error("Error uploading {}".format(media_type))
logger.exception(e)
def _stop(self):
# type: () -> None
logger.debug("VisualGridRunner.stop()")
while sum(r.score for r in self._get_all_running_tests()) > 0:
datetime_utils.sleep(500, msg="Waiting for finishing tests in stop")
self.still_running = False
for future in concurrent.futures.as_completed(self._future_to_task):
task = self._future_to_task[future]
try:
future.result()
except Exception as exc:
logger.exception("%r generated an exception: %s" % (task, exc))
else:
logger.debug("%s task ran" % task)
self.put_cache.executor.shutdown()
self.resource_cache.executor.shutdown()
self._executor.shutdown()
self._thread.join()
def poll_render_status(self, requests):
# type: (List[RenderRequest]) -> List[RenderStatusResults]
logger.debug("poll_render_status call with Requests {}".format(requests))
iterations = 0
statuses = [] # type: List[RenderStatusResults]
fails_count = 0
finished = False
while True:
if finished:
break
while True:
try:
statuses = self.eyes_connector.render_status_by_id(
*[rq.render_id for rq in requests]
)
except Exception as e:
logger.exception(e)
datetime_utils.sleep(
1000, msg="/render-status throws exception... sleeping for 1s"
)
fails_count += 1
finally:
iterations += 1
datetime_utils.sleep(1500, msg="Rendering...")
if iterations > self.MAX_ITERATIONS:
raise EyesError(
"Max iterations in poll_render_status has been reached "
"for render_id: \n {}".format(
"\n".join(s.render_id for s in statuses)
)
)
if statuses or 0 < fails_count < 3:
break
finished = bool(
statuses
and (
all(s.status != RenderStatus.RENDERING for s in statuses)
or iterations > self.MAX_ITERATIONS
or False
)
)
return statuses
def _set_viewport_size(self):
viewport_size = self.configure.viewport_size
if viewport_size is None:
for render_bi in self.configure.browsers_info:
if isinstance(render_bi, DesktopBrowserInfo) or isinstance(
render_bi, RenderBrowserInfo):
viewport_size = render_bi.viewport_size
break
if viewport_size is None:
self.configure.viewport_size = self._get_viewport_size()
return
self.configure.viewport_size = viewport_size
try:
eyes_selenium_utils.set_viewport_size(self.driver, viewport_size)
except Exception as e:
logger.exception(e)
raise EyesError(str(e))
def _get_urls_from_css_resource(resource):
def is_import_node(n):
return n.type == "at-rule" and n.lower_at_keyword == "import"
try:
rules, encoding = tinycss2.parse_stylesheet_bytes(
css_bytes=resource.content, skip_comments=True, skip_whitespace=True
)
except Exception:
logger.exception("Failed to reed CSS string")
return []
urls = []
for rule in rules:
tags = rule.content
if is_import_node(rule):
logger.debug("The node has import")
tags = rule.prelude
if tags:
urls.extend(list(_url_from_tags(tags)))
return urls
def poll_render_status(self, render_request):
# type: (RenderRequest) -> List[RenderStatusResults]
iterations = 0
statuses = []
if not render_request.render_id:
raise EyesError("RenderStatus: Got empty renderId!")
fails_count = 0
finished = False
while True:
if finished:
break
while True:
try:
statuses = self.eyes_connector.render_status_by_id(
render_request.render_id
)
except Exception as e:
logger.exception(e)
time.sleep(1)
fails_count += 1
finally:
iterations += 1
time.sleep(0.5)
if statuses or 0 < fails_count < 3:
break
finished = (
statuses
and statuses[0] is not None
and (
statuses[0].status == RenderStatus.ERROR
or statuses[0].status == RenderStatus.RENDERED
or iterations > self.MAX_ITERATIONS
or False
)
)
if statuses[0].status == RenderStatus.ERROR:
raise EyesError(
"Got error during rendering: \n\t{}".format(statuses[0].error)
)
return statuses
def check(self, name, check_settings):
# type: (Text, SeleniumCheckSettings) -> bool
if self.configuration.is_disabled:
return False
argument_guard.is_a(check_settings, CheckSettings)
logger.debug("VisualGridEyes.check(%s, %s)" % (name, check_settings))
self._try_set_target_selector(check_settings)
self.configuration.send_dom = check_settings.values.send_dom
size_mode = check_settings.values.size_mode
if size_mode is None:
if self.configuration.force_full_page_screenshot:
check_settings.values.size_mode = "full-page"
logger.info("check('{}', check_settings) - begin".format(name))
# region_xpaths = self.get_region_xpaths(check_settings)
region_xpaths = []
self.region_to_check = None
# logger.info("region_xpaths: {}".format(region_xpaths))
script_result = self.get_script_result()
try:
for test in self.test_list:
test.check(
tag=name,
check_settings=check_settings,
script_result=script_result,
visual_grid_manager=self.vg_manager,
region_selectors=region_xpaths,
size_mode=size_mode,
region_to_check=self.region_to_check,
)
if test.state == "new":
test.becomes_not_rendered()
except Exception as e:
logger.exception(e)
for test in self.test_list:
test.becomes_tested()
logger.info("added check tasks {}".format(check_settings))
def get_script_result(self):
# type: () -> Dict
script_response = {}
status = None
timer = self._start_timer()
while True:
if status == "SUCCESS" or self.is_check_timer_timeout:
self.is_check_timer_timeout = False
break
script_result_string = self.driver.execute_script(
PROCESS_RESOURCES + "return __processPageAndPoll();")
try:
script_response = json.loads(script_result_string,
object_pairs_hook=OrderedDict)
status = script_response.get("status")
except Exception as e:
logger.exception(e)
timer.cancel()
script_result = script_response.get("value")
if script_result is None or status == "ERROR":
raise EyesError("Failed to capture script_result")
return script_result
def user_agent(self):
# type: () -> useragent.UserAgent
if self.is_mobile_app:
major_version = minor_version = None
if self.platform_version:
if "." in self.platform_version:
version_parts = self.platform_version.split(".", 2)
major_version, minor_version = version_parts[:2]
else:
major_version = self.platform_version
self._user_agent = useragent.UserAgent(
os=self.platform_name,
os_major_version=major_version,
os_minor_version=minor_version,
)
if self._user_agent:
return self._user_agent
try:
ua_string = self._driver.execute_script("return navigator.userAgent")
self._user_agent = useragent.parse_user_agent_string(ua_string)
except WebDriverException as e:
logger.exception(e)
return self._user_agent
def request(self,
method,
url_resource,
use_api_key=True,
request_id=None,
**kwargs):
# type: (Text, Text, bool, Optional[UUID], **Any) -> Response
req_id = str(uuid.uuid4() if request_id is None else request_id)
if url_resource is not None:
# makes URL relative
url_resource = url_resource.lstrip("/")
url_resource = urljoin(self.server_url.rstrip("/"), url_resource)
params = {}
if use_api_key:
params["apiKey"] = self.api_key
params.update(kwargs.get("params", {}))
headers = self.headers.copy()
headers.update(kwargs.get("headers", {}))
headers["x-applitools-eyes-client-request-id"] = req_id
timeout_sec = kwargs.get("timeout", None)
if timeout_sec is None:
timeout_sec = datetime_utils.to_sec(self.timeout_ms)
response = self.client_session.request(
method,
url_resource,
data=kwargs.get("data", None),
verify=False,
params=params,
headers=headers,
timeout=timeout_sec,
)
try:
response.raise_for_status()
except requests.HTTPError as e:
logger.exception(e)
logger.error("Error response content is: {}".format(response.text))
return response
def request(self, method, url_resource, use_api_key=True, **kwargs):
if url_resource is not None:
# makes URL relative
url_resource = url_resource.lstrip("/")
url_resource = urljoin(self.server_url, url_resource)
params = {}
if use_api_key:
params["apiKey"] = self.api_key
params.update(kwargs.get("params", {}))
headers = kwargs.get("headers", self.headers).copy()
timeout = kwargs.get("timeout", self.timeout_sec)
response = method(
url_resource,
data=kwargs.get("data", None),
verify=False,
params=params,
headers=headers,
timeout=timeout,
)
try:
response.raise_for_status()
except requests.HTTPError as e:
logger.exception(e)
return response
def normalize_rotation(driver, image, rotation):
# type: (WebDriver, Image, Optional[int]) -> Image
"""
Rotates the image as necessary. The rotation is either manually forced
by passing a non-null rotation, or automatically inferred.
:param driver: The underlying driver which produced the screenshot.
:param image: The image to normalize.
:param rotation: The degrees by which to rotate the image:
positive values = clockwise rotation,
negative values = counter-clockwise,
0 = force no rotation,
null = rotate automatically as needed.
:return: A normalized image.
"""
argument_guard.not_none(driver)
argument_guard.not_none(image)
normalized_image = image
if rotation and rotation != 0:
normalized_image = image_utils.rotate_image(image, rotation)
else: # Do automatic rotation if necessary
try:
logger.info("Trying to automatically normalize rotation...")
if (
eyes_selenium_utils.is_mobile_app(driver)
and eyes_selenium_utils.is_landscape_orientation(driver)
and image.height > image.width
):
# For Android, we need to rotate images to the right,
# and for iOS to the left.
degree = 90 if eyes_selenium_utils.is_android(driver) else -90
normalized_image = image_utils.rotate_image(image, degree)
except Exception as e:
logger.exception(e)
logger.info("Skipped automatic rotation handling.")
return normalized_image
def get_dom_script_result(driver, dom_extraction_timeout, timer_name,
script_for_run):
# type: (AnyWebDriver, int, Text, Text) -> Dict
is_check_timer_timeout = []
script_response = {}
status = None
def start_timer():
def set_timer():
is_check_timer_timeout.append(True)
timer = threading.Timer(datetime_utils.to_sec(dom_extraction_timeout),
set_timer)
timer.daemon = True
timer.setName(timer_name)
timer.start()
return timer
timer = start_timer()
while True:
if status == "SUCCESS" or is_check_timer_timeout:
del is_check_timer_timeout[:]
break
script_result_string = driver.execute_script(script_for_run)
try:
script_response = json.loads(script_result_string,
object_pairs_hook=OrderedDict)
status = script_response.get("status")
except Exception as e:
logger.exception(e)
datetime_utils.sleep(1000, "Waiting for the end of DOM extraction")
timer.cancel()
script_result = script_response.get("value")
if script_result is None or status == "ERROR":
raise EyesError("Failed to capture script_result")
return script_result
def check(self, name, check_settings):
# type: (Text, SeleniumCheckSettings) -> bool
if self.configuration.is_disabled:
return False
argument_guard.is_a(check_settings, CheckSettings)
logger.debug("VisualGridEyes.check(%s, %s)" % (name, check_settings))
self._try_set_target_selector(check_settings)
self.configuration.send_dom = check_settings.values.send_dom
check_settings = self._update_check_settings(check_settings)
logger.info("check('{}', check_settings) - begin".format(name))
region_xpaths = self.get_region_xpaths(check_settings)
logger.info("region_xpaths: {}".format(region_xpaths))
script_result = self.get_script_result()
try:
for test in self.test_list:
if self._test_uuid != test.test_uuid:
continue
test.check(
tag=name,
check_settings=check_settings,
script_result=script_result,
visual_grid_manager=self.vg_manager,
region_selectors=region_xpaths,
region_to_check=check_settings.values.target_region,
script_hooks=check_settings.values.script_hooks,
)
if test.state == "new":
test.becomes_not_rendered()
except Exception as e:
logger.exception(e)
for test in self.test_list:
test.becomes_tested()
logger.info("added check tasks {}".format(check_settings))