Beispiel #1
0
    def perform(self, world):
        LOGGER.info("Performing %s" % str(self))
        screen = world.machine.automation.grab_screen()
        world.last_screen = screen.copy()

        if self.method:
            self.method(world)  # pylint: disable=E1102
        else:
            #found = self.tail.ui_element.find_in(Image2(image=screen))
            found = self.tail.find_in(Image2(image=screen))
            if found:
                #links usually need to be clicked close to the left
                #top coord (if left aligned) but some buttons needs
                #the center as they shade in the borders
                if self.ui_type == 'link':
                    x_coord = found[0] + self.location[0] + 4
                    y_coord = found[1] + self.location[1] + 4
                else:
                    center_x, center_y = center_of_rect(self.location)
                    x_coord = found[0] + center_x
                    y_coord = found[1] + center_y
                #Some quite seldom and random issues when clicking!?
                world.machine.automation.mouse.move(x_coord - 1, y_coord - 1)
                world.machine.automation.mouse.click(x_coord, y_coord)
                if self.ui_type == 'desktop icon':
                    world.machine.automation.mouse.click()
                elif self.ui_type == 'text' and 'test value' in self.custom:
                    world.machine.automation.keyboard.enters(
                        self.custom['test value'][0]['value'])
            else:
                screen.save("Current screen.bmp")
                raise Exception("Parent node (%s) not found in screen!" %
                                self.tail.name)

        #FIXME: for recording purposes this is needed, for playback purposes
        #the destination should be checked by waiting...
        world.machine.automation.mouse.move(1, 1)
        world.machine.wait_idling()
        if self.head:
            destination_reached = False
            # FIXME: this is not entirely correct, some long transitions may
            # happen for example during installations or parts of the
            # applications were it looks like stuck for a while
            for i in range(3):
                destination_reached = self.head.is_in(world)
                if destination_reached:
                    break
                LOGGER.info("Expected destination not in screen yet, "
                            "waiting (%s)" % i)
                world.machine.wait_idling()

            if destination_reached == False:
                LOGGER.info("Expected destination not in screen, could be a "
                            "type of return to caller scenario")
            elif self.head.enter_hook:
                self.head.enter_hook(world)
Beispiel #2
0
    def perform(self, world):
        LOGGER.info("Performing %s" % str(self))
        screen = world.machine.automation.grab_screen()
        world.last_screen = screen.copy()

        if self.method:
            self.method(world)  # pylint: disable=E1102
        else:
            # found = self.tail.ui_element.find_in(Image2(image=screen))
            found = self.tail.find_in(Image2(image=screen))
            if found:
                # links usually need to be clicked close to the left
                # top coord (if left aligned) but some buttons needs
                # the center as they shade in the borders
                if self.ui_type == "link":
                    x_coord = found[0] + self.location[0] + 4
                    y_coord = found[1] + self.location[1] + 4
                else:
                    center_x, center_y = center_of_rect(self.location)
                    x_coord = found[0] + center_x
                    y_coord = found[1] + center_y
                # Some quite seldom and random issues when clicking!?
                world.machine.automation.mouse.move(x_coord - 1, y_coord - 1)
                world.machine.automation.mouse.click(x_coord, y_coord)
                if self.ui_type == "desktop icon":
                    world.machine.automation.mouse.click()
                elif self.ui_type == "text" and "test value" in self.custom:
                    world.machine.automation.keyboard.enters(self.custom["test value"][0]["value"])
            else:
                screen.save("Current screen.bmp")
                raise Exception("Parent node (%s) not found in screen!" % self.tail.name)

        # FIXME: for recording purposes this is needed, for playback purposes
        # the destination should be checked by waiting...
        world.machine.automation.mouse.move(1, 1)
        world.machine.wait_idling()
        if self.head:
            destination_reached = False
            # FIXME: this is not entirely correct, some long transitions may
            # happen for example during installations or parts of the
            # applications were it looks like stuck for a while
            for i in range(3):
                destination_reached = self.head.is_in(world)
                if destination_reached:
                    break
                LOGGER.info("Expected destination not in screen yet, " "waiting (%s)" % i)
                world.machine.wait_idling()

            if destination_reached == False:
                LOGGER.info("Expected destination not in screen, could be a " "type of return to caller scenario")
            elif self.head.enter_hook:
                self.head.enter_hook(world)
Beispiel #3
0
    def get_elements_by_app_outfocus(self, hints=None, window_origin=None):
        '''
        Alternate algorithm, tries to find the elements by setting the focus
        in the desktop and then back to the application while analyzing the
        difference on the screen, when the application loses the focus the
        normal behaviour in windows is that the active control is not shown
        with the focus
        This technique has some caveats, for example the border of the
        application needs to be ignored, also the default button will also
        be rendered differently giving in some cases 2 areas with changes
        '''
        self._user_automation.mouse.move(1, 1)
        if hints is None:
            hints = {}
        if window_origin is None:
            window_origin = (0, 0)
        rects = []
        repeated = 0
        attempts = 0
        while True:
            screen = Image2(self._grab_screen())
            width = screen.size[0]
            self._user_automation.mouse.click(width / 2, screen.size[1] + 5)
            time.sleep(0.1)
            screen2 = Image2(self._grab_screen())
            self._user_automation.keyboard.enters("{+alt}{tab}{+shift}"
                                                  "{tab}{-shift}{-alt}")
            time.sleep(1)
            
            #coords = screen.difference(screen2)
            coords = automation_helpers.crop_border_differences(screen, screen2)
            crop1 = screen.image.crop((coords[0],
                                       coords[1],
                                       coords[2] + 1,
                                       coords[3] + 1))
            crop2 = screen2.image.crop((coords[0],
                                        coords[1],
                                        coords[2] + 1,
                                        coords[3] + 1))
            coords2 = Image2(image=crop1).difference(Image2(image=crop2))
            if coords2:
                coords = (coords[0] + coords2[0],
                          coords[1] + coords2[1],
                          coords[0] + coords2[2],
                          coords[1] + coords2[3])
            divisions = automation_helpers.find_inner_bbox(screen,
                                                           screen2,
                                                           coords)
            LOGGER.debug("Splitting found %s bboxes (%s)" % (len(divisions),
                                                             str(divisions)))
            repeated_in_divisions = 0
            for rect in divisions:
                print "resulting coords %s" % str(rect)
                if rect in rects:
                    repeated_in_divisions += 1
                else:
                    rects.append(rect)
            if repeated_in_divisions == len(divisions):
                repeated += 1
                if repeated == 2:
                    break
            self._user_automation.keyboard.enters("{tab}")
            time.sleep(0.1)
            attempts += 1
            if attempts - len(rects) > 5:
                #something is wrong, is possible that is unable to detect 
                #horizontal division at all
                return False
        
        result = []
        for rect in rects:
            self._user_automation.mouse.move(1, 1)
            time.sleep(0.2)
            before_cursor = self._user_automation.get_current_cursor()
            before_screen = Image2(self._grab_screen())
            center_x, center_y = center_of_rect(rect)
            self._user_automation.mouse.move(center_x, center_y)
            self._user_automation.mouse.move(center_x + 1, center_y + 1)
            time.sleep(0.2)
            after_screen = Image2(self._grab_screen())
            after_cursor = self._user_automation.get_current_cursor()

            ui_changes = before_screen != after_screen
            cursor_changes = before_cursor != after_cursor
            everything = not hints.get('visual clue needed', True)
            if cursor_changes or ui_changes or everything:
                element = {'coords': (rect[0], rect[1], rect[2], rect[3]),
                           'type': after_cursor}
                result.append(element)

        return result
Beispiel #4
0
    def get_elements(self, hints=None, window_origin=None):
        '''
        Returns a list of UI elements that this class can identify from the
        currently active window.
        The return of this implementation is an array of dictionaries where
        each dictionary describes the control type and it's bounding box
        '''
        if hints is None:
            hints = {}
        if window_origin is None:
            window_origin = (0, 0)
            
        if hints.get('outfocus method', False):
            result = self.get_elements_by_app_outfocus(hints, window_origin)
            if result != False:
                return result
            else:
                LOGGER.warning('Unable to properly use the outfocus hint, '
                               'reverting to standard behaviour for this node.')
        
        self._user_automation.mouse.move(1, 1)
            
        screen = self._grab_screen()
        screen_height = screen.size[1]
        screen = Image2(screen)
        
        tab_screens = get_tab_changing_areas(self._grab_screen,
                                             self._send_tab)
        
        if len(tab_screens) == 1:
            LOGGER.info('Only one image obtained when cycling with tab, adding'
                        ' alt trick.')
            self._user_automation.keyboard.enters('{alt}')
            #we're searching for very small clue here... just one _
            new_screen = Image2(self._grab_screen(), tolerance=1.0)
            tab_screens.append(new_screen)
            
        candidates = []
        processed = []
        for i in range(len(tab_screens)-1):
            coords = tab_screens[i].difference(tab_screens[i+1])
            if coords:
                LOGGER.debug("Changes from %s to %s are: %s" % (i,
                                                                i + 1,
                                                                str(coords)))
                division = automation_helpers.find_inner_bbox(tab_screens[i],
                                                              tab_screens[i+1],
                                                              coords)
                    
                LOGGER.debug("Splitting found %s bboxes (%s)" % (len(division),
                                                                 str(division)))
                for rect in division:
                    if not rect in candidates:
                        LOGGER.debug("Adding: %s" % str(rect))
                        candidates.append(rect)
                        #hover, if image differs take diff coords, use biggest
                        # of two use mouse pointer clue for type
                        #ARGGGGGG cursor blinking... deactivated at os level for
                        #now
                        
                        #the focus may be at this point anywhere and on 1st
                        #case is where it is left from tab navigation, for
                        #cases like menu we have to highlight current menu item
                        center = center_of_rect(rect)
                        self._user_automation.mouse.move(center[0], center[1])
                        self._user_automation.mouse.move(center[0]+1,
                                                         center[1]+1)
                        cursor = self._user_automation.get_current_cursor()
                        screen1 = Image2(self._grab_screen())
                        self._user_automation.mouse.move(1, screen_height)
                        screen2 = Image2(self._grab_screen())
                        diff = screen1.difference(screen2)
                            
                        if diff: #produced a change in UI, must be button
                            LOGGER.debug(("Will compute biggest rect out of "
                                          "%s %s") % (str(rect), str(diff)))
                            biggest_rect = get_bounding_box([rect, diff])
                            if not biggest_rect in processed:
                                processed.append(biggest_rect)
                                LOGGER.debug("Added: %s" % str(biggest_rect))
                        else:
                            #no UI change, can be a link, text or misfired
                            #recognition, exceptional case is one button alone
                            #in dialog
                            if ((cursor != 'normal' and not rect in processed) or
                              (len(tab_screens) == 2 and not rect in processed)):
                                processed.append(rect)
                                LOGGER.debug("Added: %s" % str(rect))
        
        LOGGER.debug("There are %s elements to consider from tab + hovering" %
                     len(processed))
        
        checkboxes = self._checkboxes.find_all(screen)
        LOGGER.debug("Found %s checkboxes" % len(checkboxes))
        checkboxes = add_text_to_elements(checkboxes, screen)

        radios = self._radio.find_all(screen)
        LOGGER.debug("Found %s radios" % len(checkboxes))
        radios = add_text_to_elements(radios, screen)

        checkboxes = merge_overlapping_areas(checkboxes, processed)
        radios = merge_overlapping_areas(radios, processed)
        
        areas = exclude_subareas(processed, checkboxes + radios)
        
        points = hints.get('points of interest', [])
        LOGGER.debug("Points of interest are: %s" % str(points))
        for point in points:
            point_x = window_origin[0] + point[0]
            point_y = window_origin[1] + point[1]
            found, bbox = find_bounding_box(screen.image, point_x, point_y)
            if found:
                LOGGER.debug("Found %s from point of interest" % str(bbox))
                areas.append(bbox)
            else:
                LOGGER.debug("Nothing found from point of interest at %s %s" %
                             (point_x, point_y))
        
        result = []
        for area in areas:
            center_x, center_y = center_of_rect(area)
            self._user_automation.mouse.move(center_x, center_y)
            self._user_automation.mouse.move(center_x + 1, center_y + 1)
            cursor = self._user_automation.get_current_cursor()
            element = {'coords': (area[0], area[1], area[2], area[3]),
                       'type': cursor}
            result.append(element)
            
        for area in checkboxes:
            element = {'coords': (area[0], area[1], area[2], area[3]),
                       'type': 'checkbox'}
            result.append(element)

        for area in radios:
            element = {'coords': (area[0], area[1], area[2], area[3]),
                       'type': 'radio'}
            result.append(element)

        result = remove_containers(result)
        return result
Beispiel #5
0
    def get_elements_by_app_outfocus(self, hints=None, window_origin=None):
        '''
        Alternate algorithm, tries to find the elements by setting the focus
        in the desktop and then back to the application while analyzing the
        difference on the screen, when the application loses the focus the
        normal behaviour in windows is that the active control is not shown
        with the focus
        This technique has some caveats, for example the border of the
        application needs to be ignored, also the default button will also
        be rendered differently giving in some cases 2 areas with changes
        '''
        self._user_automation.mouse.move(1, 1)
        if hints is None:
            hints = {}
        if window_origin is None:
            window_origin = (0, 0)
        rects = []
        repeated = 0
        attempts = 0
        while True:
            screen = Image2(self._grab_screen())
            width = screen.size[0]
            self._user_automation.mouse.click(width / 2, screen.size[1] + 5)
            time.sleep(0.1)
            screen2 = Image2(self._grab_screen())
            self._user_automation.keyboard.enters("{+alt}{tab}{+shift}"
                                                  "{tab}{-shift}{-alt}")
            time.sleep(1)

            #coords = screen.difference(screen2)
            coords = automation_helpers.crop_border_differences(
                screen, screen2)
            crop1 = screen.image.crop(
                (coords[0], coords[1], coords[2] + 1, coords[3] + 1))
            crop2 = screen2.image.crop(
                (coords[0], coords[1], coords[2] + 1, coords[3] + 1))
            coords2 = Image2(image=crop1).difference(Image2(image=crop2))
            if coords2:
                coords = (coords[0] + coords2[0], coords[1] + coords2[1],
                          coords[0] + coords2[2], coords[1] + coords2[3])
            divisions = automation_helpers.find_inner_bbox(
                screen, screen2, coords)
            LOGGER.debug("Splitting found %s bboxes (%s)" %
                         (len(divisions), str(divisions)))
            repeated_in_divisions = 0
            for rect in divisions:
                print "resulting coords %s" % str(rect)
                if rect in rects:
                    repeated_in_divisions += 1
                else:
                    rects.append(rect)
            if repeated_in_divisions == len(divisions):
                repeated += 1
                if repeated == 2:
                    break
            self._user_automation.keyboard.enters("{tab}")
            time.sleep(0.1)
            attempts += 1
            if attempts - len(rects) > 5:
                #something is wrong, is possible that is unable to detect
                #horizontal division at all
                return False

        result = []
        for rect in rects:
            self._user_automation.mouse.move(1, 1)
            time.sleep(0.2)
            before_cursor = self._user_automation.get_current_cursor()
            before_screen = Image2(self._grab_screen())
            center_x, center_y = center_of_rect(rect)
            self._user_automation.mouse.move(center_x, center_y)
            self._user_automation.mouse.move(center_x + 1, center_y + 1)
            time.sleep(0.2)
            after_screen = Image2(self._grab_screen())
            after_cursor = self._user_automation.get_current_cursor()

            ui_changes = before_screen != after_screen
            cursor_changes = before_cursor != after_cursor
            everything = not hints.get('visual clue needed', True)
            if cursor_changes or ui_changes or everything:
                element = {
                    'coords': (rect[0], rect[1], rect[2], rect[3]),
                    'type': after_cursor
                }
                result.append(element)

        return result
Beispiel #6
0
    def get_elements(self, hints=None, window_origin=None):
        '''
        Returns a list of UI elements that this class can identify from the
        currently active window.
        The return of this implementation is an array of dictionaries where
        each dictionary describes the control type and it's bounding box
        '''
        if hints is None:
            hints = {}
        if window_origin is None:
            window_origin = (0, 0)

        if hints.get('outfocus method', False):
            result = self.get_elements_by_app_outfocus(hints, window_origin)
            if result != False:
                return result
            else:
                LOGGER.warning(
                    'Unable to properly use the outfocus hint, '
                    'reverting to standard behaviour for this node.')

        self._user_automation.mouse.move(1, 1)

        screen = self._grab_screen()
        screen_height = screen.size[1]
        screen = Image2(screen)

        tab_screens = get_tab_changing_areas(self._grab_screen, self._send_tab)

        if len(tab_screens) == 1:
            LOGGER.info('Only one image obtained when cycling with tab, adding'
                        ' alt trick.')
            self._user_automation.keyboard.enters('{alt}')
            #we're searching for very small clue here... just one _
            new_screen = Image2(self._grab_screen(), tolerance=1.0)
            tab_screens.append(new_screen)

        candidates = []
        processed = []
        for i in range(len(tab_screens) - 1):
            coords = tab_screens[i].difference(tab_screens[i + 1])
            if coords:
                LOGGER.debug("Changes from %s to %s are: %s" %
                             (i, i + 1, str(coords)))
                division = automation_helpers.find_inner_bbox(
                    tab_screens[i], tab_screens[i + 1], coords)

                LOGGER.debug("Splitting found %s bboxes (%s)" %
                             (len(division), str(division)))
                for rect in division:
                    if not rect in candidates:
                        LOGGER.debug("Adding: %s" % str(rect))
                        candidates.append(rect)
                        #hover, if image differs take diff coords, use biggest
                        # of two use mouse pointer clue for type
                        #ARGGGGGG cursor blinking... deactivated at os level for
                        #now

                        #the focus may be at this point anywhere and on 1st
                        #case is where it is left from tab navigation, for
                        #cases like menu we have to highlight current menu item
                        center = center_of_rect(rect)
                        self._user_automation.mouse.move(center[0], center[1])
                        self._user_automation.mouse.move(
                            center[0] + 1, center[1] + 1)
                        cursor = self._user_automation.get_current_cursor()
                        screen1 = Image2(self._grab_screen())
                        self._user_automation.mouse.move(1, screen_height)
                        screen2 = Image2(self._grab_screen())
                        diff = screen1.difference(screen2)

                        if diff:  #produced a change in UI, must be button
                            LOGGER.debug(("Will compute biggest rect out of "
                                          "%s %s") % (str(rect), str(diff)))
                            biggest_rect = get_bounding_box([rect, diff])
                            if not biggest_rect in processed:
                                processed.append(biggest_rect)
                                LOGGER.debug("Added: %s" % str(biggest_rect))
                        else:
                            #no UI change, can be a link, text or misfired
                            #recognition, exceptional case is one button alone
                            #in dialog
                            if ((cursor != 'normal' and not rect in processed)
                                    or (len(tab_screens) == 2
                                        and not rect in processed)):
                                processed.append(rect)
                                LOGGER.debug("Added: %s" % str(rect))

        LOGGER.debug("There are %s elements to consider from tab + hovering" %
                     len(processed))

        checkboxes = self._checkboxes.find_all(screen)
        LOGGER.debug("Found %s checkboxes" % len(checkboxes))
        checkboxes = add_text_to_elements(checkboxes, screen)

        radios = self._radio.find_all(screen)
        LOGGER.debug("Found %s radios" % len(checkboxes))
        radios = add_text_to_elements(radios, screen)

        checkboxes = merge_overlapping_areas(checkboxes, processed)
        radios = merge_overlapping_areas(radios, processed)

        areas = exclude_subareas(processed, checkboxes + radios)

        points = hints.get('points of interest', [])
        LOGGER.debug("Points of interest are: %s" % str(points))
        for point in points:
            point_x = window_origin[0] + point[0]
            point_y = window_origin[1] + point[1]
            found, bbox = find_bounding_box(screen.image, point_x, point_y)
            if found:
                LOGGER.debug("Found %s from point of interest" % str(bbox))
                areas.append(bbox)
            else:
                LOGGER.debug("Nothing found from point of interest at %s %s" %
                             (point_x, point_y))

        result = []
        for area in areas:
            center_x, center_y = center_of_rect(area)
            self._user_automation.mouse.move(center_x, center_y)
            self._user_automation.mouse.move(center_x + 1, center_y + 1)
            cursor = self._user_automation.get_current_cursor()
            element = {
                'coords': (area[0], area[1], area[2], area[3]),
                'type': cursor
            }
            result.append(element)

        for area in checkboxes:
            element = {
                'coords': (area[0], area[1], area[2], area[3]),
                'type': 'checkbox'
            }
            result.append(element)

        for area in radios:
            element = {
                'coords': (area[0], area[1], area[2], area[3]),
                'type': 'radio'
            }
            result.append(element)

        result = remove_containers(result)
        return result