def add_text_to_elements(rects, image):
'''
Returns a rect that includes the given rect and it's acompanying text at
the right of it
'''
including_text = []
for rect in rects:
found, left, top, right, bottom = find_element_text(rect,
image,
right_whites=5)
if found:
including_text.append(
get_bounding_box([rect, (left, top, right, bottom)]))
return including_text
def add_text_to_elements(rects, image):
'''
Returns a rect that includes the given rect and it's acompanying text at
the right of it
'''
including_text = []
for rect in rects:
found, left, top, right, bottom = find_element_text(rect,
image,
right_whites=5)
if found:
including_text.append(get_bounding_box([rect,
(left,
top,
right,
bottom)]))
return including_text
def get_elements(self, hints=None, window_origin=None):
'''
Returns a list of UI elements that this class can identify from the
currently active window.
The return of this implementation is an array of dictionaries where
each dictionary describes the control type and it's bounding box
'''
if hints is None:
hints = {}
if window_origin is None:
window_origin = (0, 0)
if hints.get('outfocus method', False):
result = self.get_elements_by_app_outfocus(hints, window_origin)
if result != False:
return result
else:
LOGGER.warning('Unable to properly use the outfocus hint, '
'reverting to standard behaviour for this node.')
self._user_automation.mouse.move(1, 1)
screen = self._grab_screen()
screen_height = screen.size[1]
screen = Image2(screen)
tab_screens = get_tab_changing_areas(self._grab_screen,
self._send_tab)
if len(tab_screens) == 1:
LOGGER.info('Only one image obtained when cycling with tab, adding'
' alt trick.')
self._user_automation.keyboard.enters('{alt}')
#we're searching for very small clue here... just one _
new_screen = Image2(self._grab_screen(), tolerance=1.0)
tab_screens.append(new_screen)
candidates = []
processed = []
for i in range(len(tab_screens)-1):
coords = tab_screens[i].difference(tab_screens[i+1])
if coords:
LOGGER.debug("Changes from %s to %s are: %s" % (i,
i + 1,
str(coords)))
division = automation_helpers.find_inner_bbox(tab_screens[i],
tab_screens[i+1],
coords)
LOGGER.debug("Splitting found %s bboxes (%s)" % (len(division),
str(division)))
for rect in division:
if not rect in candidates:
LOGGER.debug("Adding: %s" % str(rect))
candidates.append(rect)
#hover, if image differs take diff coords, use biggest
# of two use mouse pointer clue for type
#ARGGGGGG cursor blinking... deactivated at os level for
#now
#the focus may be at this point anywhere and on 1st
#case is where it is left from tab navigation, for
#cases like menu we have to highlight current menu item
center = center_of_rect(rect)
self._user_automation.mouse.move(center[0], center[1])
self._user_automation.mouse.move(center[0]+1,
center[1]+1)
cursor = self._user_automation.get_current_cursor()
screen1 = Image2(self._grab_screen())
self._user_automation.mouse.move(1, screen_height)
screen2 = Image2(self._grab_screen())
diff = screen1.difference(screen2)
if diff: #produced a change in UI, must be button
LOGGER.debug(("Will compute biggest rect out of "
"%s %s") % (str(rect), str(diff)))
biggest_rect = get_bounding_box([rect, diff])
if not biggest_rect in processed:
processed.append(biggest_rect)
LOGGER.debug("Added: %s" % str(biggest_rect))
else:
#no UI change, can be a link, text or misfired
#recognition, exceptional case is one button alone
#in dialog
if ((cursor != 'normal' and not rect in processed) or
(len(tab_screens) == 2 and not rect in processed)):
processed.append(rect)
LOGGER.debug("Added: %s" % str(rect))
LOGGER.debug("There are %s elements to consider from tab + hovering" %
len(processed))
checkboxes = self._checkboxes.find_all(screen)
LOGGER.debug("Found %s checkboxes" % len(checkboxes))
checkboxes = add_text_to_elements(checkboxes, screen)
radios = self._radio.find_all(screen)
LOGGER.debug("Found %s radios" % len(checkboxes))
radios = add_text_to_elements(radios, screen)
checkboxes = merge_overlapping_areas(checkboxes, processed)
radios = merge_overlapping_areas(radios, processed)
areas = exclude_subareas(processed, checkboxes + radios)
points = hints.get('points of interest', [])
LOGGER.debug("Points of interest are: %s" % str(points))
for point in points:
point_x = window_origin[0] + point[0]
point_y = window_origin[1] + point[1]
found, bbox = find_bounding_box(screen.image, point_x, point_y)
if found:
LOGGER.debug("Found %s from point of interest" % str(bbox))
areas.append(bbox)
else:
LOGGER.debug("Nothing found from point of interest at %s %s" %
(point_x, point_y))
result = []
for area in areas:
center_x, center_y = center_of_rect(area)
self._user_automation.mouse.move(center_x, center_y)
self._user_automation.mouse.move(center_x + 1, center_y + 1)
cursor = self._user_automation.get_current_cursor()
element = {'coords': (area[0], area[1], area[2], area[3]),
'type': cursor}
result.append(element)
for area in checkboxes:
element = {'coords': (area[0], area[1], area[2], area[3]),
'type': 'checkbox'}
result.append(element)
for area in radios:
element = {'coords': (area[0], area[1], area[2], area[3]),
'type': 'radio'}
result.append(element)
result = remove_containers(result)
return result
def get_elements(self, hints=None, window_origin=None):
'''
Returns a list of UI elements that this class can identify from the
currently active window.
The return of this implementation is an array of dictionaries where
each dictionary describes the control type and it's bounding box
'''
if hints is None:
hints = {}
if window_origin is None:
window_origin = (0, 0)
if hints.get('outfocus method', False):
result = self.get_elements_by_app_outfocus(hints, window_origin)
if result != False:
return result
else:
LOGGER.warning(
'Unable to properly use the outfocus hint, '
'reverting to standard behaviour for this node.')
self._user_automation.mouse.move(1, 1)
screen = self._grab_screen()
screen_height = screen.size[1]
screen = Image2(screen)
tab_screens = get_tab_changing_areas(self._grab_screen, self._send_tab)
if len(tab_screens) == 1:
LOGGER.info('Only one image obtained when cycling with tab, adding'
' alt trick.')
self._user_automation.keyboard.enters('{alt}')
#we're searching for very small clue here... just one _
new_screen = Image2(self._grab_screen(), tolerance=1.0)
tab_screens.append(new_screen)
candidates = []
processed = []
for i in range(len(tab_screens) - 1):
coords = tab_screens[i].difference(tab_screens[i + 1])
if coords:
LOGGER.debug("Changes from %s to %s are: %s" %
(i, i + 1, str(coords)))
division = automation_helpers.find_inner_bbox(
tab_screens[i], tab_screens[i + 1], coords)
LOGGER.debug("Splitting found %s bboxes (%s)" %
(len(division), str(division)))
for rect in division:
if not rect in candidates:
LOGGER.debug("Adding: %s" % str(rect))
candidates.append(rect)
#hover, if image differs take diff coords, use biggest
# of two use mouse pointer clue for type
#ARGGGGGG cursor blinking... deactivated at os level for
#now
#the focus may be at this point anywhere and on 1st
#case is where it is left from tab navigation, for
#cases like menu we have to highlight current menu item
center = center_of_rect(rect)
self._user_automation.mouse.move(center[0], center[1])
self._user_automation.mouse.move(
center[0] + 1, center[1] + 1)
cursor = self._user_automation.get_current_cursor()
screen1 = Image2(self._grab_screen())
self._user_automation.mouse.move(1, screen_height)
screen2 = Image2(self._grab_screen())
diff = screen1.difference(screen2)
if diff: #produced a change in UI, must be button
LOGGER.debug(("Will compute biggest rect out of "
"%s %s") % (str(rect), str(diff)))
biggest_rect = get_bounding_box([rect, diff])
if not biggest_rect in processed:
processed.append(biggest_rect)
LOGGER.debug("Added: %s" % str(biggest_rect))
else:
#no UI change, can be a link, text or misfired
#recognition, exceptional case is one button alone
#in dialog
if ((cursor != 'normal' and not rect in processed)
or (len(tab_screens) == 2
and not rect in processed)):
processed.append(rect)
LOGGER.debug("Added: %s" % str(rect))
LOGGER.debug("There are %s elements to consider from tab + hovering" %
len(processed))
checkboxes = self._checkboxes.find_all(screen)
LOGGER.debug("Found %s checkboxes" % len(checkboxes))
checkboxes = add_text_to_elements(checkboxes, screen)
radios = self._radio.find_all(screen)
LOGGER.debug("Found %s radios" % len(checkboxes))
radios = add_text_to_elements(radios, screen)
checkboxes = merge_overlapping_areas(checkboxes, processed)
radios = merge_overlapping_areas(radios, processed)
areas = exclude_subareas(processed, checkboxes + radios)
points = hints.get('points of interest', [])
LOGGER.debug("Points of interest are: %s" % str(points))
for point in points:
point_x = window_origin[0] + point[0]
point_y = window_origin[1] + point[1]
found, bbox = find_bounding_box(screen.image, point_x, point_y)
if found:
LOGGER.debug("Found %s from point of interest" % str(bbox))
areas.append(bbox)
else:
LOGGER.debug("Nothing found from point of interest at %s %s" %
(point_x, point_y))
result = []
for area in areas:
center_x, center_y = center_of_rect(area)
self._user_automation.mouse.move(center_x, center_y)
self._user_automation.mouse.move(center_x + 1, center_y + 1)
cursor = self._user_automation.get_current_cursor()
element = {
'coords': (area[0], area[1], area[2], area[3]),
'type': cursor
}
result.append(element)
for area in checkboxes:
element = {
'coords': (area[0], area[1], area[2], area[3]),
'type': 'checkbox'
}
result.append(element)
for area in radios:
element = {
'coords': (area[0], area[1], area[2], area[3]),
'type': 'radio'
}
result.append(element)
result = remove_containers(result)
return result
