def _wait(self, type, loc):
'''
ID = "id"
XPATH = "xpath"
LINK_TEXT = "link text"
PARTIAL_LINK_TEXT = "partial link text"
NAME = "name"
TAG_NAME = "tag name"
CLASS_NAME = "class name"
CSS_SELECTOR = "css selector"
'''
try:
if type == 'xpath':
WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.XPATH, loc)))
elif type == 'id':
WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.ID, loc)))
elif type == 'css':
WebDriverWait(self.driver, 20).until(
EC.presence_of_element_located((By.CSS_SELECTOR, loc)))
# return self.driver.find_element_by_xpath(loc).click()
except (NoSuchElementException, TimeoutException):
logger.error('[{}]寻找元素失败, 定位方式为{}:{}'.format(
os.path.basename(__file__), type, loc))
raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为{}:{}'.format(
os.path.basename(__file__), type, loc))
def test_regist_001(self):
"""新用户注册"""
logger.info('*======================用例' +
sys._getframe().f_code.co_name +
'开始执行======================*')
one_browser_window()
sql = "delete from user_info where phone='18637607203';"
flag = mysql.update(sql)
if flag:
logger.info("用户删除成功")
else:
logger.error("用户删除失败")
self.loginpage.open_regist()
self.loginpage.regist('gaozuxin', 'gzx123456', '18637607203', '9274',
'A1b8S')
self.loginpage.take_screenshot()
islogin = self.indexpage.get_loginuser()
if islogin == 'gaozuxin':
logger.info('注册并登录成功')
else:
logger.error('注册并登录失败')
time.sleep(2)
self.indexpage.click_exit_userbtn()
time.sleep(3)
logger.info('*======================用例' +
sys._getframe().f_code.co_name +
'执行结束======================*')
def click_select_category(self,
category): # category1：选择规则_常规采集；category2：
if category == 'category1':
self.find_element(self.select_category1_loc).click()
else:
logger.error('[{}]未找到指定规则分类:{}'.format(os.path.basename(__file__),
category))
raise NoSuchElementException(msg='[{}]未找到指定规则:{}'.format(
os.path.basename(__file__), category))
def wrapper(*args, **kw):
for i in range(n):
try:
r = func(*args, **kw)
return r
except Exception as err:
logger.error('[{}]用例第{}次失败原因{}'.format(
os.path.basename(__file__), i + 1, err))
raise Exception
def select_element(self, element_xpath, select_text):
select1 = self.find_element(element_xpath)
try:
Select(select1).select_by_visible_text(select_text)
logger.info("[{}]选择的元素是:{}".format(os.path.basename(__file__),
select_text))
except (NoSuchElementException, TimeoutException) as e:
logger.error("[{}]无法选择该元素{}".format(os.path.basename(__file__), e))
raise NoSuchElementException(
msg='[{}]无法选择该元素{}'.format(os.path.basename(__file__), e))
def select_rule_type(self, rule_type): # type1：页面类型_列表页；type2：页面类型_正文页
if rule_type == 'type1':
self.find_element(self.rule_type1_loc).click()
elif rule_type == 'type2':
self.find_element(self.rule_type2_loc).click()
else:
logger.error('[{}]未找到指定页面类型:{}'.format(os.path.basename(__file__),
rule_type))
raise NoSuchElementException(msg='[{}]未找到指定页面类型:{}'.format(
os.path.basename(__file__), rule_type))
def take_screenshot(self):
rq = time.strftime("%Y-%m-%d %H_%M_%S", time.localtime())
try:
self._driver.get_screenshot_as_file(
os.path.dirname(os.path.dirname(__file__)) +
'\screenshots\{}.png'.format(rq))
logger.info("[{}]已截屏并保存{}.png!".format(os.path.basename(__file__),
rq))
except Exception as e:
logger.error("[{}]无法截屏!{}".format(os.path.basename(__file__), e))
def select_extract_method(
self, extract_method): # method1:配置类型_可视化点选;method2:配置类型_自定义点选
if extract_method == 'method1':
self.find_element(self.extract_method1_loc).click()
elif extract_method == 'method2':
self.find_element(self.extract_method2_loc).click()
else:
logger.error('[{}]未找到指定配置类型:{}'.format(os.path.basename(__file__),
extract_method))
raise NoSuchElementException(msg='[{}]未找到指定配置类型:{}'.format(
os.path.basename(__file__), extract_method))
def find_element(self, *loc):
try:
WebDriverWait(self._driver,
20).until(lambda driver: driver.
find_element_by_xpath(*loc).is_displayed())
return self._driver.find_element_by_xpath(*loc)
except (NoSuchElementException, TimeoutException):
# self._driver.quit()
logger.error('[{}]寻找元素失败, 定位方式为xpath:{}'.format(
os.path.basename(__file__), loc))
raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为xpath:{}'.format(
os.path.basename(__file__), loc))
def send_keys(self, el, vaule, clear_first=True, click_first=True):
try:
if click_first:
el.click()
if clear_first:
el.clear()
el.send_keys(vaule)
except (NoSuchElementException, TimeoutException) as e:
# NoSuchElementException:
logger.error("[{}]页面中未能找到元素{}".format(os.path.basename(__file__),
e))
raise NoSuchElementException(
msg='[{}]页面中未能找到元素{}'.format(os.path.basename(__file__), e))
def click_select_category(
self,
category): # category1：常规；category2：微信；category3：微博；category4：递归
if category == 'category1':
self.find_element(self.select_category1_loc).click()
elif category == 'category2':
self.find_element(self.select_category2_loc).click()
elif category == 'category3':
self.find_element(self.select_category3_loc).click()
elif category == 'category4':
self.find_element(self.select_category4_loc).click()
else:
logger.error("未找到指定规则分类")
def get_conn(self):
try:
conn = pymysql.connect(host=self.host,
user=self.user,
passwd=self.password,
db=self.db,
port=self.port,
charset='utf8')
except pymysql.Error as e:
logger.error("[{}]connect database failed.{}".format(
os.path.basename(__file__), e))
conn = False
return conn
def new_regular_collection(self, urllist_lis, select_extractrule_list,
check1_list, check2_list, turnpagec_list,
opentimeout_list, waittime_list, sampleurl_text,
selcet_extractrule_text, turnpagec_text,
opentimeout_text, waittime_text):
self.click_element(self.piderstype1_loc)
self.click_element(self.listcollec_loc)
self.input_listurl_list(urllist_lis)
time.sleep(1) # 加载提取规则
try:
self.leftclick(self.extractrule_list_loc)
time.sleep(6)
self.select_element(self.extractrule_list_loc,
select_extractrule_list)
except:
logger.error("[{}]提取规则加载失败".format(os.path.basename(__file__)))
time.sleep(1)
self.leftclick(self.extractrule_list_loc)
time.sleep(6)
self.select_element(self.extractrule_list_loc,
select_extractrule_list)
time.sleep(1)
self.select_element(self.double_check1_list_loc, check1_list)
self.select_element(self.double_check2_list_loc, check2_list)
self.input_turn_pagec_list(turnpagec_list)
self.select_element(self.opentimeout_list_loc, opentimeout_list)
self.select_element(self.waittime_list_loc, waittime_list)
self.click_element(self.textcollec_loc)
self.input_sampleurl_text(sampleurl_text)
self.click_element(self.extracttype2_text_loc)
time.sleep(1) # 加载提取规则
try:
self.leftclick(self.extractrule_text_loc)
time.sleep(6)
self.select_element(self.extractrule_text_loc,
selcet_extractrule_text)
except:
logger.error("[{}]提取规则加载失败".format(os.path.basename(__file__)))
time.sleep(1)
self.leftclick(self.extractrule_text_loc)
time.sleep(6)
self.select_element(self.extractrule_text_loc,
select_extractrule_list)
time.sleep(1)
self.input_turn_pagec_text(turnpagec_text)
self.select_element(self.opentimeout_text_loc, opentimeout_text)
self.select_element(self.waittime_text_loc, waittime_text)
def _loading(self, type, loc):
try:
# loading出现
WebDriverWait(self.driver, 30).until(
EC.visibility_of_element_located(
(By.ID, 'layui-layer-shade1')))
# loading消失
WebDriverWait(self.driver, 30).until_not(
EC.visibility_of_element_located(
(By.ID, 'layui-layer-shade1')))
except (NoSuchElementException, TimeoutException):
# self._driver.quit()
logger.error('[{}]寻找元素失败, 定位方式为{}:{}'.format(
os.path.basename(__file__), type, loc))
raise TimeoutException(msg='[{}]寻找元素失败, 定位方式为{}:{}'.format(
os.path.basename(__file__), type, loc))
def test005_task(self):
"""微博采集_微博号采集"""
logger.info('*======================用例' +
sys._getframe().f_code.co_name +
'开始执行======================*')
one_browser_window()
sql1 = "delete from task_info where uid = 163 and task_name = '微博采集_微博号_大数据';"
flag1 = mysql.update(sql1)
if flag1 == 1:
logger.info('微博采集微博号任务删除成功')
else:
logger.error('微博采集微博号任务删除失败')
self.loginpage.open_login()
self.loginpage.login('18201112814', 'gzx123456', 'A1b8S')
self.loginpage.take_screenshot()
self.loginpage.find_element(Taskherf_loc).click()
self.taskpage.add_task()
time.sleep(2)
handles = driver.window_handles
driver.switch_to.window(handles[-1])
taskname = '微博采集_微博号_大数据'
taskcategory = 'category2'
tasktype = '调度任务'
cron = '*/1 * * * *'
timebreak_loc = '30'
isagent = '否'
self.addtask.collect_init(taskname, taskcategory, tasktype, cron,
timebreak_loc, isagent)
wbkeyword = '大数据'
self.addtask.new_weibo_collection(wbkeyword)
time.sleep(2)
self.addtask.click_savecnfbtn1()
self.addtask.click_confirmbtn()
self.addrule.take_screenshot()
time.sleep(5)
self.addtask.click_effectivenowbtn1()
self.loginpage.take_screenshot()
driver.switch_to.window(handles[0])
sql3 = "select task_id from task_info where uid = 163 and task_name = '微博采集_微博号_大数据';"
task_id1 = mysql.select_one(sql3)
time.sleep(120)
self.loginpage.find_element(Dataherf_loc).click()
task_id2 = self.datapage.get_first_taskid()
self.loginpage.take_screenshot()
if str(task_id1[0]) == str(task_id2):
self.datapage.click_see_details()
tasks = self.datapage.get_first_tasks()
logger.info("任务抓取条数{}".format(tasks))
if str(tasks) != '0':
logger.info("任务数据抓取正常")
else:
logger.error("任务数据抓取异常")
else:
logger.error("任务id获取异常")
logger.info('*======================用例' +
sys._getframe().f_code.co_name +
'执行结束======================*')
def get_result(self, count_num):
break_flag = False
n = count_num // 10 + 1
m = count_num % 10 + 1
for i in range(1, n):
for index in range(1, 11):
if i == n - 1 and index == m:
break_flag = True
break
rule_name_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[2]/span'.format(
index)
testbtn_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[10]/a[4]'.format(
index)
type_loc = '//*[@id="contentheight"]/div/table/tbody/tr[{}]/td[3]/span'.format(
index)
rule_name = self.find_element(rule_name_loc).text
type = self.find_element(type_loc).text
time.sleep(1)
self.click_element(testbtn_loc)
time.sleep(20)
if type == '列表页':
result = self.find_element(self.result_loc).text
if result == '共0条数据':
errinfo = self.find_element(self.errinfo_loc).text
logger.error("规则名称:(第{}页第{}个){},{},#{}".format(
i, index, rule_name, result, errinfo))
else:
logger.info("规则名称:(第{}页第{}个){},{}".format(
i, index, rule_name, result))
time.sleep(2)
elif type == '正文页':
try:
title_text = self.find_element(
self.title_text_loc).text
content_text = self.find_element(
self.content_text_loc).text
time.sleep(2)
logger.info(
"规则名称:(第{}页第{}个){},title:{};content:{}".format(
i, index, rule_name, title_text, content_text))
except:
logger.error("规则名称:(第{}页第{}个)页面数据获取有误".format(
i, index, rule_name))
else:
logger.error('所属类型有误')
self.take_screenshot()
self.click_element(self.closebtn_loc)
if break_flag:
break
time.sleep(1)
# self..click_element(next_page_loc)
self.next_page(i + 1)
def test002_task(self):
"""用户登录、配置规则、添加任务"""
logger.info('*===============用例{}\{}开始执行===============*'.format(
__name__,
sys._getframe().f_code.co_name))
one_browser_window()
sql1 = "delete from task_rule where uid = 163 and rule_name like '新建规则_%页';"
sql2 = "delete from task_info where uid = 163 and task_name like '新建任务_非异步';"
flag1 = mysql.update(sql1)
flag2 = mysql.update(sql2)
if flag1 == 1 and flag2 == 1:
logger.info('任务和规则删除成功')
else:
logger.error('任务和规则删除失败')
# driver.set_window_size(900, 600)
# window_size = driver.get_window_size()
# logger.info(window_size)
#
# time.sleep(2)
# driver.maximize_window()
# window_size = driver.get_window_size()
# logger.info(window_size)
#
# time.sleep(2)
# driver.set_window_size(3000, 1000)
# window_size = driver.get_window_size()
# logger.info(window_size)
# driver.set_window_size(1000, 812)
# window_size = driver.get_window_size()
# logger.info(window_size)
self.loginpage.open_login()
self.loginpage.login('18201112814', 'gzx123456', 'A1b8S')
self.loginpage.take_screenshot()
try:
self.loginpage.find_element(Ruleherf_loc).click()
except:
self.loginpage.take_screenshot()
exit()
self.rulepage.add_rule()
time.sleep(1)
handles = driver.window_handles # 获取所有打开窗口的句柄
driver.switch_to.window(handles[1])
rulename1 = '新建规则_列表页'
select_category1 = 'category1'
rule_type1 = 'type1'
pageurl1 = 'http://in.nen.com.cn/ssjj/index.shtml'
url_pattern1 = 'http://in.nen.com.cn*'
rulemark1 = 'test_list'
extract_method1 = 'method1'
self.addrule.click_enter_piontconf(rulename1, select_category1,
rule_type1, pageurl1, url_pattern1,
rulemark1, extract_method1)
time.sleep(1)
handles = driver.window_handles
driver.switch_to.window(handles[-1])
self.extensionpage.add_dongbeinews_listrule()
self.addrule.click_savecnewrulebtn()
self.addrule.take_screenshot()
time.sleep(5)
rulename2 = '新建规则_正文页'
select_category2 = 'category1'
rule_type2 = 'type2'
pageurl2 = 'http://in.nen.com.cn/system/2018/03/23/020430991.shtml'
url_pattern2 = 'http://in.nen.com.cn*'
rulemark2 = 'test_text'
extract_method2 = 'method1'
self.addrule.click_enter_piontconf(rulename2, select_category2,
rule_type2, pageurl2, url_pattern2,
rulemark2, extract_method2)
time.sleep(2)
handles = driver.window_handles
driver.switch_to.window(handles[-1])
self.extensionpage.add_dongbeinews_textrule()
self.addrule.click_saverulebtn()
self.addrule.take_screenshot()
driver.switch_to.window(handles[0])
self.loginpage.find_element(Taskherf_loc).click()
self.taskpage.add_task()
time.sleep(2)
handles = driver.window_handles
driver.switch_to.window(handles[-1])
taskname = '新建任务_非异步'
taskcategory = 'category1'
tasktype = '调度任务'
cron = '*/1 * * * *'
timebreak_loc = '30'
isagent = '否'
self.addtask.collect_init(taskname, taskcategory, tasktype, cron,
timebreak_loc, isagent)
urllist_lis = 'http://in.nen.com.cn/ssjj/index.shtml'
select_extractrule_list = "新建规则_列表页"
check1_list = 'url'
check2_list = '当选中字段已经存在时跳过'
turnpagec_list = '0'
opentimeout_list = '5秒'
waittime_list = '不等待'
sampleurl_text = 'http://in.nen.com.cn/system/2018/03/23/020430991.shtml'
selcet_extractrule_text = "新建规则_正文页"
turnpagec_text = '0'
opentimeout_text = '5秒'
waittime_text = '不等待'
self.addtask.new_regular_collection(
urllist_lis, select_extractrule_list, check1_list, check2_list,
turnpagec_list, opentimeout_list, waittime_list, sampleurl_text,
selcet_extractrule_text, turnpagec_text, opentimeout_text,
waittime_text)
self.addtask.click_savecnfbtn()
self.addtask.click_confirmbtn()
self.addrule.take_screenshot()
time.sleep(2)
self.addtask.click_effectivenowbtn()
self.loginpage.take_screenshot()
driver.switch_to.window(handles[0])
sql3 = "select task_id from task_info where uid = 163 and task_name = '新建任务_非异步';"
task_id1 = mysql.select_one(sql3)
time.sleep(220)
self.loginpage.find_element(Dataherf_loc).click()
task_id2 = self.datapage.get_first_taskid()
self.loginpage.take_screenshot()
if str(task_id1[0]) == str(task_id2):
self.datapage.click_see_details()
tasks = self.datapage.get_first_tasks()
logger.info("任务抓取条数{}".format(tasks))
if str(tasks) == '30':
logger.info("任务数据抓取正常")
else:
logger.error("任务数据抓取异常")
else:
logger.error("任务id获取异常")
logger.info('*======================用例' +
sys._getframe().f_code.co_name +
'执行结束======================*')
