def scroll_smooth(self, x_dist, y_dist, x_neg, y_neg, two_paned_scroll='window', element=True): try: src_str = 'window' if two_paned_scroll != src_str: if element: src_str = ''.join([ 'document.getElementById(\'', two_paned_scroll, '\')' ]) else: src_str = ''.join([ 'document.getElementsByClassName(\'', two_paned_scroll, '\')[0]' ]) scroll_x = '.scrollBy(-8,0);' if x_neg else '.scrollBy(8,0);' scroll_y = '.scrollBy(0,-8);' if y_neg else '.scrollBy(0,8);' for i in range(x_dist // 8): self.driver.execute_script(src_str + scroll_x) rand_block(0, 0.001) for i in range(y_dist // 8): self.driver.execute_script(src_str + scroll_y) rand_block(0, 0.001) except WebDriverException as e: log.warning(e) raise
def scroll_by_key_down(self, dat, distance_togo): """ 模拟 page_up/down 方式来滚动 :param dat: :type dat: :param distance_togo: :type distance_togo: :return: :rtype: """ try: target = self.get_elements(dat) distance_done = 0 while distance_done < distance_togo: chance = random.random() if chance < 0.05: rand_block(0.5, 0.1) continue distance = rand_pareto_float(100, 10) if chance >= 0.95: distance = -distance key_press_times_ = int(distance // PIXELS_PER_DOWN_KEY) self.mock_key_down(target, key_press_times_) distance_done += key_press_times_ * PIXELS_PER_DOWN_KEY except Exception as _: log.error(_) raise
def scroll_to_top(self): try: _scroll = 'window.scrollTo(0,0);' self.driver.execute_script(_scroll) rand_block(0, 0.001) except WebDriverException as e: log.error(e.msg) raise
def goto_next(self, yval=0): self.pre_goto_next() # if can goes here, self.next page shoud not be none!!! assert self.next_page is not None self.robot.scroll_to_element(self.next_page) rand_block(1, 0.5, (3, 5)) self.robot.mock_click_next_page(self.next_page) self.wait_page_load() return yval
def _mock_input(self, target, content): """ mock human input :param target: the element to input to :param content: the content :return: """ content = helper.to_str(content) for w in content: target.send_keys(w) rand_block(0.01, 0.01)
def mock_key_down(self, target, key_press_times): """ key_press_times => >= 0 则 DOWN, 否则 UP :param target: :type target: :param key_press_times: :type key_press_times: int :return: :rtype: """ key = Keys.DOWN if key_press_times < 0: key = Keys.UP for i in range(abs(key_press_times)): rand_block(0.1, 0.01) target.send_keys(key)
def mock_input(self, dat, term='', clear=True, mock_input_change='', submit=False, multiple=0): """ 输入 term 到 input:ele(dat) 元素 如果 ``mock_input_change`` 不为空, 则该 input 不能保留原始值, 可以输入一个值, 然后清空. - clear 清空input的元素 - submit 提交 :param dat: :type dat: dict :param term: :type term: str :param submit: :type submit: bool :param clear: :type clear: bool :param mock_input_change: 针对不能保留原来值的输入框 :type mock_input_change: str :return: :rtype: """ target = self.get_elements(dat, multiple) if multiple: target = target[multiple] if clear: target.clear() self._mock_input(target, Keys.BACKSPACE * 50) if mock_input_change: self._mock_input(target, mock_input_change) rand_block(0.01, 0.01) self._mock_input(target, Keys.BACKSPACE * (len(mock_input_change) + 2)) if term: self._mock_input(target, term) if submit: rand_block(0.01, 0.1) target.send_keys(Keys.ENTER) rand_block(0.01, 0.1)
def mock_popovers(self): for d in self.base.get('popovers', []): if self.robot.has_element(d, skip_log=True): rand_block(1, 1, slow_mode=(2, 5)) self.robot.mock_click(d) break
def wait_page_load(self): st = time.time() rand_block(1, 0.5, (2, 10)) log.debug('{} takes {}s for next page {}'.format( '>' * 16, int(time.time() - st), '<' * 16))
def scroll_to_element(self, dat, two_paned_scroll='window', element=True, close_in=0, speed=1, check_time=True, total_down=0, ret=False): # scroll down to element target = self.get_elements(dat) # 已完成距离 distance_done = total_down # 待完成距离 w_size = self.driver.get_window_size() # log.debug('y total need to go: {}'.format(target.location['y'])) # only let y visable!!! distance_togo = target.location['y'] - w_size['height'] + 150 # distance_togo = target.location['y'] - 300 # y // <num> 防止滚动过多 if not close_in: close_in = w_size['height'] // 3 starting_time = time.time() distance = 0 while distance_done < distance_togo: if (time.time() - starting_time) > 150 and check_time: raise TooSlowScrollException chance = random.random() if chance < 0.1: # 在0.1 的概率下等待一定时间 mini_ = min(0.01 / chance, 0.5) rand_block(mini_, 0.1) else: # 以很小的概率向上 scroll if chance > 0.96: y_neg, distance = True, -rand_pareto_float(100, 10) else: # 多数情况下向下 scroll y_neg, distance = False, rand_pareto_float(100, 10) * speed # log.debug('Each time: {}/{}/{}'.format(distance_togo, distance_done, distance)) # distance_togo + close_in distance = min(distance_togo - distance_done + close_in, distance) distance_done += distance self.scroll_smooth(0, int(abs(distance)), False, y_neg, two_paned_scroll, element) else: dist_ = (distance_done - distance_togo - w_size['height'] // 3) // 2 log.debug('done togo: {}-{}={}'.format(int(distance_done), int(distance_togo), dist_)) if dist_ > 0: log.debug('scroll back {} '.format(dist_)) self.scroll_smooth(0, int(abs(dist_)), False, True, two_paned_scroll, element) rand_block(1, 1, (2, 3)) if ret: return distance_togo
def mock_click_next_page(self, dat): btn = self.get_elements(dat) rand_block(0.1, 0.2) return self.mock_click(btn)