def _wait(self, type, loc): ''' ID = "id" XPATH = "xpath" LINK_TEXT = "link text" PARTIAL_LINK_TEXT = "partial link text" NAME = "name" TAG_NAME = "tag name" CLASS_NAME = "class name" CSS_SELECTOR = "css selector" ''' try: if type == 'xpath': WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.XPATH, loc))) elif type == 'id': WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.ID, loc))) elif type == 'css': WebDriverWait(self.driver, 20).until( EC.presence_of_element_located((By.CSS_SELECTOR, loc))) # return self.driver.find_element_by_xpath(loc).click() except (NoSuchElementException, TimeoutException): logger.error('[{}]寻找元素失败, 定位方式为{}:{}'.format( os.path.basename(__file__), type, loc)) raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为{}:{}'.format( os.path.basename(__file__), type, loc))
def test_regist_001(self): """新用户注册""" logger.info('*======================用例' + sys._getframe().f_code.co_name + '开始执行======================*') one_browser_window() sql = "delete from user_info where phone='18637607203';" flag = mysql.update(sql) if flag: logger.info("用户删除成功") else: logger.error("用户删除失败") self.loginpage.open_regist() self.loginpage.regist('gaozuxin', 'gzx123456', '18637607203', '9274', 'A1b8S') self.loginpage.take_screenshot() islogin = self.indexpage.get_loginuser() if islogin == 'gaozuxin': logger.info('注册并登录成功') else: logger.error('注册并登录失败') time.sleep(2) self.indexpage.click_exit_userbtn() time.sleep(3) logger.info('*======================用例' + sys._getframe().f_code.co_name + '执行结束======================*')
def click_select_category(self, category): # category1:选择规则_常规采集;category2: if category == 'category1': self.find_element(self.select_category1_loc).click() else: logger.error('[{}]未找到指定规则分类:{}'.format(os.path.basename(__file__), category)) raise NoSuchElementException(msg='[{}]未找到指定规则:{}'.format( os.path.basename(__file__), category))
def wrapper(*args, **kw): for i in range(n): try: r = func(*args, **kw) return r except Exception as err: logger.error('[{}]用例第{}次失败原因{}'.format( os.path.basename(__file__), i + 1, err)) raise Exception
def select_element(self, element_xpath, select_text): select1 = self.find_element(element_xpath) try: Select(select1).select_by_visible_text(select_text) logger.info("[{}]选择的元素是:{}".format(os.path.basename(__file__), select_text)) except (NoSuchElementException, TimeoutException) as e: logger.error("[{}]无法选择该元素{}".format(os.path.basename(__file__), e)) raise NoSuchElementException( msg='[{}]无法选择该元素{}'.format(os.path.basename(__file__), e))
def select_rule_type(self, rule_type): # type1:页面类型_列表页;type2:页面类型_正文页 if rule_type == 'type1': self.find_element(self.rule_type1_loc).click() elif rule_type == 'type2': self.find_element(self.rule_type2_loc).click() else: logger.error('[{}]未找到指定页面类型:{}'.format(os.path.basename(__file__), rule_type)) raise NoSuchElementException(msg='[{}]未找到指定页面类型:{}'.format( os.path.basename(__file__), rule_type))
def take_screenshot(self): rq = time.strftime("%Y-%m-%d %H_%M_%S", time.localtime()) try: self._driver.get_screenshot_as_file( os.path.dirname(os.path.dirname(__file__)) + '\screenshots\{}.png'.format(rq)) logger.info("[{}]已截屏并保存{}.png!".format(os.path.basename(__file__), rq)) except Exception as e: logger.error("[{}]无法截屏!{}".format(os.path.basename(__file__), e))
def select_extract_method( self, extract_method): # method1:配置类型_可视化点选;method2:配置类型_自定义点选 if extract_method == 'method1': self.find_element(self.extract_method1_loc).click() elif extract_method == 'method2': self.find_element(self.extract_method2_loc).click() else: logger.error('[{}]未找到指定配置类型:{}'.format(os.path.basename(__file__), extract_method)) raise NoSuchElementException(msg='[{}]未找到指定配置类型:{}'.format( os.path.basename(__file__), extract_method))
def find_element(self, *loc): try: WebDriverWait(self._driver, 20).until(lambda driver: driver. find_element_by_xpath(*loc).is_displayed()) return self._driver.find_element_by_xpath(*loc) except (NoSuchElementException, TimeoutException): # self._driver.quit() logger.error('[{}]寻找元素失败, 定位方式为xpath:{}'.format( os.path.basename(__file__), loc)) raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为xpath:{}'.format( os.path.basename(__file__), loc))
def send_keys(self, el, vaule, clear_first=True, click_first=True): try: if click_first: el.click() if clear_first: el.clear() el.send_keys(vaule) except (NoSuchElementException, TimeoutException) as e: # NoSuchElementException: logger.error("[{}]页面中未能找到元素{}".format(os.path.basename(__file__), e)) raise NoSuchElementException( msg='[{}]页面中未能找到元素{}'.format(os.path.basename(__file__), e))
def click_select_category( self, category): # category1:常规;category2:微信;category3:微博;category4:递归 if category == 'category1': self.find_element(self.select_category1_loc).click() elif category == 'category2': self.find_element(self.select_category2_loc).click() elif category == 'category3': self.find_element(self.select_category3_loc).click() elif category == 'category4': self.find_element(self.select_category4_loc).click() else: logger.error("未找到指定规则分类")
def get_conn(self): try: conn = pymysql.connect(host=self.host, user=self.user, passwd=self.password, db=self.db, port=self.port, charset='utf8') except pymysql.Error as e: logger.error("[{}]connect database failed.{}".format( os.path.basename(__file__), e)) conn = False return conn
def new_regular_collection(self, urllist_lis, select_extractrule_list, check1_list, check2_list, turnpagec_list, opentimeout_list, waittime_list, sampleurl_text, selcet_extractrule_text, turnpagec_text, opentimeout_text, waittime_text): self.click_element(self.piderstype1_loc) self.click_element(self.listcollec_loc) self.input_listurl_list(urllist_lis) time.sleep(1) # 加载提取规则 try: self.leftclick(self.extractrule_list_loc) time.sleep(6) self.select_element(self.extractrule_list_loc, select_extractrule_list) except: logger.error("[{}]提取规则加载失败".format(os.path.basename(__file__))) time.sleep(1) self.leftclick(self.extractrule_list_loc) time.sleep(6) self.select_element(self.extractrule_list_loc, select_extractrule_list) time.sleep(1) self.select_element(self.double_check1_list_loc, check1_list) self.select_element(self.double_check2_list_loc, check2_list) self.input_turn_pagec_list(turnpagec_list) self.select_element(self.opentimeout_list_loc, opentimeout_list) self.select_element(self.waittime_list_loc, waittime_list) self.click_element(self.textcollec_loc) self.input_sampleurl_text(sampleurl_text) self.click_element(self.extracttype2_text_loc) time.sleep(1) # 加载提取规则 try: self.leftclick(self.extractrule_text_loc) time.sleep(6) self.select_element(self.extractrule_text_loc, selcet_extractrule_text) except: logger.error("[{}]提取规则加载失败".format(os.path.basename(__file__))) time.sleep(1) self.leftclick(self.extractrule_text_loc) time.sleep(6) self.select_element(self.extractrule_text_loc, select_extractrule_list) time.sleep(1) self.input_turn_pagec_text(turnpagec_text) self.select_element(self.opentimeout_text_loc, opentimeout_text) self.select_element(self.waittime_text_loc, waittime_text)
def _loading(self, type, loc): try: # loading出现 WebDriverWait(self.driver, 30).until( EC.visibility_of_element_located( (By.ID, 'layui-layer-shade1'))) # loading消失 WebDriverWait(self.driver, 30).until_not( EC.visibility_of_element_located( (By.ID, 'layui-layer-shade1'))) except (NoSuchElementException, TimeoutException): # self._driver.quit() logger.error('[{}]寻找元素失败, 定位方式为{}:{}'.format( os.path.basename(__file__), type, loc)) raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为{}:{}'.format( os.path.basename(__file__), type, loc))
def test005_task(self): """微博采集_微博号采集""" logger.info('*======================用例' + sys._getframe().f_code.co_name + '开始执行======================*') one_browser_window() sql1 = "delete from task_info where uid = 163 and task_name = '微博采集_微博号_大数据';" flag1 = mysql.update(sql1) if flag1 == 1: logger.info('微博采集微博号任务删除成功') else: logger.error('微博采集微博号任务删除失败') self.loginpage.open_login() self.loginpage.login('18201112814', 'gzx123456', 'A1b8S') self.loginpage.take_screenshot() self.loginpage.find_element(Taskherf_loc).click() self.taskpage.add_task() time.sleep(2) handles = driver.window_handles driver.switch_to.window(handles[-1]) taskname = '微博采集_微博号_大数据' taskcategory = 'category2' tasktype = '调度任务' cron = '*/1 * * * *' timebreak_loc = '30' isagent = '否' self.addtask.collect_init(taskname, taskcategory, tasktype, cron, timebreak_loc, isagent) wbkeyword = '大数据' self.addtask.new_weibo_collection(wbkeyword) time.sleep(2) self.addtask.click_savecnfbtn1() self.addtask.click_confirmbtn() self.addrule.take_screenshot() time.sleep(5) self.addtask.click_effectivenowbtn1() self.loginpage.take_screenshot() driver.switch_to.window(handles[0]) sql3 = "select task_id from task_info where uid = 163 and task_name = '微博采集_微博号_大数据';" task_id1 = mysql.select_one(sql3) time.sleep(120) self.loginpage.find_element(Dataherf_loc).click() task_id2 = self.datapage.get_first_taskid() self.loginpage.take_screenshot() if str(task_id1[0]) == str(task_id2): self.datapage.click_see_details() tasks = self.datapage.get_first_tasks() logger.info("任务抓取条数{}".format(tasks)) if str(tasks) != '0': logger.info("任务数据抓取正常") else: logger.error("任务数据抓取异常") else: logger.error("任务id获取异常") logger.info('*======================用例' + sys._getframe().f_code.co_name + '执行结束======================*')
def get_result(self, count_num): break_flag = False n = count_num // 10 + 1 m = count_num % 10 + 1 for i in range(1, n): for index in range(1, 11): if i == n - 1 and index == m: break_flag = True break rule_name_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[2]/span'.format( index) testbtn_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[10]/a[4]'.format( index) type_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[3]/span'.format( index) rule_name = self.find_element(rule_name_loc).text type = self.find_element(type_loc).text time.sleep(1) self.click_element(testbtn_loc) time.sleep(20) if type == '列表页': result = self.find_element(self.result_loc).text if result == '共0条数据': errinfo = self.find_element(self.errinfo_loc).text logger.error("规则名称:(第{}页第{}个){},{},#{}".format( i, index, rule_name, result, errinfo)) else: logger.info("规则名称:(第{}页第{}个){},{}".format( i, index, rule_name, result)) time.sleep(2) elif type == '正文页': try: title_text = self.find_element( self.title_text_loc).text content_text = self.find_element( self.content_text_loc).text time.sleep(2) logger.info( "规则名称:(第{}页第{}个){},title:{};content:{}".format( i, index, rule_name, title_text, content_text)) except: logger.error("规则名称:(第{}页第{}个)页面数据获取有误".format( i, index, rule_name)) else: logger.error('所属类型有误') self.take_screenshot() self.click_element(self.closebtn_loc) if break_flag: break time.sleep(1) # self..click_element(next_page_loc) self.next_page(i + 1)
def test002_task(self): """用户登录、配置规则、添加任务""" logger.info('*===============用例{}\{}开始执行===============*'.format( __name__, sys._getframe().f_code.co_name)) one_browser_window() sql1 = "delete from task_rule where uid = 163 and rule_name like '新建规则_%页';" sql2 = "delete from task_info where uid = 163 and task_name like '新建任务_非异步';" flag1 = mysql.update(sql1) flag2 = mysql.update(sql2) if flag1 == 1 and flag2 == 1: logger.info('任务和规则删除成功') else: logger.error('任务和规则删除失败') # driver.set_window_size(900, 600) # window_size = driver.get_window_size() # logger.info(window_size) # # time.sleep(2) # driver.maximize_window() # window_size = driver.get_window_size() # logger.info(window_size) # # time.sleep(2) # driver.set_window_size(3000, 1000) # window_size = driver.get_window_size() # logger.info(window_size) # driver.set_window_size(1000, 812) # window_size = driver.get_window_size() # logger.info(window_size) self.loginpage.open_login() self.loginpage.login('18201112814', 'gzx123456', 'A1b8S') self.loginpage.take_screenshot() try: self.loginpage.find_element(Ruleherf_loc).click() except: self.loginpage.take_screenshot() exit() self.rulepage.add_rule() time.sleep(1) handles = driver.window_handles # 获取所有打开窗口的句柄 driver.switch_to.window(handles[1]) rulename1 = '新建规则_列表页' select_category1 = 'category1' rule_type1 = 'type1' pageurl1 = 'http://in.nen.com.cn/ssjj/index.shtml' url_pattern1 = 'http://in.nen.com.cn*' rulemark1 = 'test_list' extract_method1 = 'method1' self.addrule.click_enter_piontconf(rulename1, select_category1, rule_type1, pageurl1, url_pattern1, rulemark1, extract_method1) time.sleep(1) handles = driver.window_handles driver.switch_to.window(handles[-1]) self.extensionpage.add_dongbeinews_listrule() self.addrule.click_savecnewrulebtn() self.addrule.take_screenshot() time.sleep(5) rulename2 = '新建规则_正文页' select_category2 = 'category1' rule_type2 = 'type2' pageurl2 = 'http://in.nen.com.cn/system/2018/03/23/020430991.shtml' url_pattern2 = 'http://in.nen.com.cn*' rulemark2 = 'test_text' extract_method2 = 'method1' self.addrule.click_enter_piontconf(rulename2, select_category2, rule_type2, pageurl2, url_pattern2, rulemark2, extract_method2) time.sleep(2) handles = driver.window_handles driver.switch_to.window(handles[-1]) self.extensionpage.add_dongbeinews_textrule() self.addrule.click_saverulebtn() self.addrule.take_screenshot() driver.switch_to.window(handles[0]) self.loginpage.find_element(Taskherf_loc).click() self.taskpage.add_task() time.sleep(2) handles = driver.window_handles driver.switch_to.window(handles[-1]) taskname = '新建任务_非异步' taskcategory = 'category1' tasktype = '调度任务' cron = '*/1 * * * *' timebreak_loc = '30' isagent = '否' self.addtask.collect_init(taskname, taskcategory, tasktype, cron, timebreak_loc, isagent) urllist_lis = 'http://in.nen.com.cn/ssjj/index.shtml' select_extractrule_list = "新建规则_列表页" check1_list = 'url' check2_list = '当选中字段已经存在时跳过' turnpagec_list = '0' opentimeout_list = '5秒' waittime_list = '不等待' sampleurl_text = 'http://in.nen.com.cn/system/2018/03/23/020430991.shtml' selcet_extractrule_text = "新建规则_正文页" turnpagec_text = '0' opentimeout_text = '5秒' waittime_text = '不等待' self.addtask.new_regular_collection( urllist_lis, select_extractrule_list, check1_list, check2_list, turnpagec_list, opentimeout_list, waittime_list, sampleurl_text, selcet_extractrule_text, turnpagec_text, opentimeout_text, waittime_text) self.addtask.click_savecnfbtn() self.addtask.click_confirmbtn() self.addrule.take_screenshot() time.sleep(2) self.addtask.click_effectivenowbtn() self.loginpage.take_screenshot() driver.switch_to.window(handles[0]) sql3 = "select task_id from task_info where uid = 163 and task_name = '新建任务_非异步';" task_id1 = mysql.select_one(sql3) time.sleep(220) self.loginpage.find_element(Dataherf_loc).click() task_id2 = self.datapage.get_first_taskid() self.loginpage.take_screenshot() if str(task_id1[0]) == str(task_id2): self.datapage.click_see_details() tasks = self.datapage.get_first_tasks() logger.info("任务抓取条数{}".format(tasks)) if str(tasks) == '30': logger.info("任务数据抓取正常") else: logger.error("任务数据抓取异常") else: logger.error("任务id获取异常") logger.info('*======================用例' + sys._getframe().f_code.co_name + '执行结束======================*')