def request_data_home(self):
"""构造首页请求数据"""
start_time = self.start_time
end_time = self.end_time
self.data = \
{
"qdi": self.apiqdi,
"starttime": start_time,
"begintime": start_time,
'begindate': start_time,
"endtime": end_time,
"enddate": end_time,
"datetype": self.apidatatype,
"rootcategoryid": None,
"groupcode": getattr(self, "groupcode", None),
"hours": "00:00-00:59",
"token": self.token,
"tagid":None,
"vspid":"1",
"selectdate": getattr(self, "request_code", None),
"channelcode":getattr(self, "request_code", None),
"sysid": getattr(self,"secsiono","u"),
"userid": "test1",
}
if getattr(self, "movetype","") == "全网频道":
self.data["groupcode"] = None
log().debug("requests 'url, data' \t {1},{0}".format(self.data, self.url_api))
def api_result(self):
"""发送请求并返回数据"""
headr = {
"Accept": "application/json, text/javascript, */*; q=0.01",
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
}
try:
res = requests.post(self.url_api, data=self.data, headers=headr, timeout=bt.wait_ele-4)
text = eval(res.text)
except NameError:
"""str>>dict"""
text = json.loads(res.text)
except ReadTimeout as e:
"""Timeout"""
self.e=e
text = dict()
self.datatext = list()
log().error(e)
self.datatext=text.get("subject", [])[0:2]
log().info("result subject data is {1}\t ".format(text, self.datatext))
if self.datatext:
Ioput.input("datatext", self.datatext)
else:
raise AssertionError(r"statplatform5 output not data ,test stop \n[error :{0}] \n request data is: {1}".format(getattr(self, "e", None), self.data))
def keyword_search(self, ):
"""关键字搜索"""
data = self.table_firstline_list
name1 = [i[0] for i in data]
number = 0
while True:
"""随机关键字"""
randomnamepice = random.choice(name1)
sindex = random.randint(0, len(randomnamepice))
eindex = random.randint(0, len(randomnamepice))
if sindex < eindex:
keyword = randomnamepice[sindex:eindex]
break
elif number >= 100:
"""100次不合法 置空"""
keyword = ""
break
number += 1
log().debug("keyword is {0}".format(keyword))
kw_ele_id = "%sSearch" % self.table_id
self.exement(elements=kw_ele_id, ele_type="id", parms=keyword)
data = self._get_tabledata(tablenum="2", getlines=20)[1]
log().debug("after search keywords {0}".format(data))
self.screenhost_to_report(alt="关键字搜索", name="关键字搜索")
for i in data:
if keyword not in i[0]:
raise AssertionError("关键字 : \"%s\" %s 在查询的结果中不存在" % (keyword, i))
def set_rule(self):
"""设置待匹配的字段，规则"""
for i in self.rule:
sql_init = "INSERT INTO WS_RULE VALUES('%d','%s','%s','%s')" % (self.rule.index(i), i[0], i[0], i[1])
log().debug(sql_init)
db_oracle(sql_init)
def move_detaildata(self):
"""上下移动，截屏"""
log().debug("滚动到屏幕顶部")
"""滚动到屏幕底部获取详情数据，并截屏"""
scroll_top_js = "window.scrollTo(0,0);" #顶端
scroll_end_js = "window.scrollTo(0,document.body.scrollHeight);" #底部
scroll_view_js = "arguments[0].scrollIntoView();" #指定元素
time.sleep(0.2)
ele_top = ""
classname = self.__class__.__name__
if hasattr(self, "text") and self.text == "查询":
ele_top = self.driver.find_elements_by_id("searchButton")
elif hasattr(self, "p") and self.p == "groupToggle":
ele_top = self.driver.find_elements_by_id("groupToggle")
elif hasattr(self, "up_text"):
ele_top = self.driver.find_elements(**self.up_text)
elif "详情" in classname:
try:
ele_top = self.driver.find_elements_by_id("nameTitle")
except:
ele_top = self.driver.find_elements_by_id("select2-changetime-container")
if ele_top:
self.driver.execute_script(scroll_view_js, ele_top[0]) # 移动到 顶端
name = os.path.splitext(self.dlfile)[0]+"表"
self.screenhost_to_report(alt=name+"header", name=name+"header")
log().debug("滚动到屏幕底部")
eledown = self.driver.find_elements(**self.down_text)
if eledown:
self.driver.execute_script(scroll_view_js, eledown[0]) #移动到指定元素 底部
def _set_time_day(self):
"""日报"""
log().debug("日报")
try:
self.exement("day", "id")
except AssertionError as a:
if "分钟收视" not in self.__class__.__name__ :
raise AssertionError(a)
starttime = self.starttime
endtime = self.endtime
sy = starttime.split("-")[0]
sm = str(int(starttime.split("-")[1]) - 1)
ey = endtime.split("-")[0]
em = str(int(endtime.split("-")[1]) - 1)
self.exement("starttime","id")
self.exement('//*[@class="layui-laydate"]//*[@lay-type="year"]')
self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % sy)
self.exement('//*[@class="layui-laydate"]//*[@lay-type="month"]')
self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % sm)
self.exement('//*[@class="layui-laydate"]//*[@lay-ymd="%s"]' % starttime)
self.exement("endtime", "id")
self.exement('//*[@class="layui-laydate"]//*[@lay-type="year"]')
self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % ey)
self.exement('//*[@class="layui-laydate"]//*[@lay-type="month"]')
self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % em)
self.exement('//*[@class="layui-laydate"]//*[@lay-ymd="%s"]' % endtime)
def activity(self, s):
"""更改ws_process表，触发融合操作"""
for i in self.code:
sql="update WS_PROCESS set PROCESSRESULT='%s' where code='%s'" % (s,i)
db_oracle(sql)
log().debug([i, db_oracle("select PROCESSRESULT from WS_PROCESS where code='%s' " % i)])
self.status = "stop"
def get_tablefirstdata(self):
"""获取详情数据的第一行"""
log().debug(self.table_id)
self.wait_element(self.table_id, "id")
self.ele_Table = self.exement(self.table_id, "id", "getele") #self.driver.find_element_by_id(self.table_id)
self.ele_tables = self.ele_Table.find_elements_by_xpath('//div[starts-with(@class,"table-")]')
self.data_lines, self.table_firstline_list = self._get_tabledata("2")
log().info("table content is %s" % self.table_firstline_list)
def count_user(self):
""""""
self.wait_element(self.table_id, "id")
element = self.driver.find_element_by_id(self.table_id)
text = element.text
if text == "0" or text == "0%":
log().error("表 {0} 无数据 end".format(self.table_id))
self.getsysereorlog("end")
raise AssertionError("表 {0} 无数据 end".format(self.table_id))
def init(self):
"""初始化环境"""
strcode = str(self.id_list).replace("[", "").replace("]", "")
clean_side= "delete WS_OUTSIDE where id in ({0})".format(strcode)
db_oracle(clean_side)
clean_tail = "delete WS_PROCESSDETAIL where id in ({0})".format(strcode)
db_oracle(clean_tail)
log().info("clean WS_PROCESSDETAIL and WS_OUTSIDE sucess")
def _set_time_week(self):
"""月报"""
log().debug("周报")
self.exement('//*[@id="week"]')
self.exement('//*[@id="select2-starttime-container"]')
self.exement('//*[@id="select2-starttime-results"]//li[text()="{0}"]'.format(self.starttime))
self.exement('//*[@id="select2-endtime-container"]')
self.exement('//*[@id="select2-endtime-results"]//li[text()="{0}"]'.format(self.endtime))
def order_top5(self):
"""top5"""
patten='//*[@id="{0}"]/div[1]/canvas'.format(self.table_id)
self.wait_element('%s[1]' % patten)
elements = self.driver.find_elements_by_xpath('//*[@id="{0}"]/div[1]/canvas'.format(self.table_id))
if not elements:
log().error("表 {0} 无数据 end".format(self.table_id))
self.getsysereorlog("end")
raise AssertionError("表 {0} 无数据 end".format(self.table_id))
def open(self):
"""打开浏览器"""
options = webdriver.ChromeOptions()
prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': BI_protal.downloadpath}
options.add_experimental_option('prefs', prefs)
options.add_argument('log-level=3')
self.driver = webdriver.Chrome(executable_path=BI_protal.driver_path,chrome_options=options)
self.driver.maximize_window()
self.driver.implicitly_wait(BI_protal.wait_time)
log().info("{0}启动浏览器{0}".format("*" * 40))
def wait_element(self, elementstr, element_type="xpath", seq=0.5):
"""显示等待"""
self.elementstr = elementstr
self.element_type = element_type
log().info("轮询元素 %s : %s " % (self.element_type, self.elementstr))
try:
WebDriverWait(self.driver, self.alltime, seq).until(self)
except:
log().error("超时，timeout >= %s秒 元素 <%s : %s> 未加载；" % (self.alltime, element_type, elementstr))
raise AssertionError("超时，timeout >= %s 元素 <%s : %s> 未加载；" % (self.alltime, element_type, elementstr))
def test_c2c3(self):
"""未存在C3数据时，爬取扩展数据+融合数据"""
I.function_name(self.__class__.__name__)
try:
obj = GetC3Data(**{'rule': [('YEAR', '1')]})
obj.execute_c2c3()
except AssertionError as e:
log().error(e)
self.assertTrue("")
else:
self.assertEqual(obj.status, "pass")
def look(self):
"""点击第一条的查看，切入详情页"""
log().debug("点击‘查看’，进入详情页")
"""点击查看，并切换到frame"""
try:
self.ele_Table.find_elements_by_xpath('//a[text()="查看"]')[0].click()
except:
log().error("详情数据表中，无数据")
raise AssertionError("无数据")
self.driver.switch_to.parent_frame()
self.frame_change(-1)
def executeapi(self, url, data={}):
self.replacepar(url=url)
if data:
self.result = requests.post(url=self.url,
data=data,
timeout=self.timeout)
elif not data:
self.result = requests.get(url=self.url, timeout=self.timeout)
log().info("\n{1}{0}".format(self.result.text, "\t" * 2))
log().info(self.result.url)
def get_text(self):
e = []
try:
try:
self.text = eval(self.result.text)
except Exception as e1:
e.append(e1)
self.text = json.loads(self.result.text)
except Exception as e2:
e.append(e2)
log().error("不合发的数据")
raise Exception(e)
def __searchsql(self):
"""根据qdi匹配日志中的sql"""
lastlist = self.alllist[ListenServerLog.Numbers:]
strs = "".join(lastlist)
p1 = r"\d.+?%s:.+?(\[oracle.+?\])\n+" % self.qdi
p2 = r"\d.+?%s:.+?Preparing: (select.+?)\n+" % self.qdi
p3 = r"\d.+?%s:.+?Parameters: ((?:.+\(.+?\), ){0,}(?:.+\(.+?\)))\n" % self.qdi
pattern = p1 + p2 + p3
self.list = re.compile(pattern=pattern).findall(strs)
log().info("%s search count %s = %s" %
(self.status, self.qdi, len(self.list)))
def test_exitsc3(self):
"""已有C3数据时爬去扩展数据"""
I.function_name(self.__class__.__name__)
try:
obj = GetC3Data()
obj.execute_exitsc3()
except AssertionError as e:
log().error(e)
self.assertTrue("")
else:
self.assertEqual(obj.status, "pass")
def test_idpc(self, kwargs):
"""测试融合规则"""
I.function_name(self.__class__.__name__,
getdata("data.txt", "idpc").index(kwargs) + 1)
try:
obj = Medial(**kwargs)
obj.execute_c2()
except AssertionError as e:
log().error(e)
self.assertTrue("", e)
else:
self.assertEqual(obj.status, "pass")
def search_c3(self):
"""查询C3数据"""
strcode = str(self.code).replace("[", "").replace("]", "")
sql_id = "select ID FROM WS_PROCESSDETAIL where SOURCE='3' and C2CODE in ({0})".format(strcode)
resid = db_oracle(sql_id)
id_list=[i[0] for i in resid]
strid_list = str(id_list).replace("[", "").replace("]", "")
sql_outside="SELECT count(*) from WS_OUTSIDE where ID in ({0})".format(strid_list)
res_side=db_oracle(sql=sql_outside)
self.id_list=id_list
self.side=res_side[0][0]
log().info("%s WS_PROCESSDETAIL ID have %s, WS_OUTSIDE ID have %s " % (self.status ,self.id_list, self.side))
def change_sec(self):
"""电信、移动、联通节点选择"""
self.wait_element('//*[@id="div-select"]/form/span/label', "xpath")
sec_text = self.driver.find_element_by_xpath('//*[@id="div-select"]/form/span/label').text
sec_text = sec_text.strip()
log().debug((sec_text, type(sec_text)))
d = {"中国电信": "1", "中国移动": "2", "中国联通": "3"}
if d[sec_text] != self.sec_text:
self.exement('//*[@id="div-select"]/form/span')
time.sleep(0.5)
self.exement('//*[@id="div-select"]/form/ul/li[%s]/a' % self.sec_text)
time.sleep(0.5)
log().info("landing %s sucess , secaction %s" % (self.url, self.sec_text))
def pararms(self, **kwargs):
"""设置参数"""
log().debug("%s" % kwargs)
self.timetype = kwargs.get("timetype", self.time_type)
self.starttime = self.Times[self.timetype].get("starttime", bt.starttime)
self.endtime = self.Times[self.timetype].get("endtime", bt.endtime)
self.apidatatype = self.Times[self.timetype].get("datatype","D")
self.user_index = kwargs.get("user_index", "1")
self.table_id = kwargs.get("table_id", "userDevelopTable")
self.dlfile = kwargs.get("dlfile", "用户发展数据.xlsx")
self.apiqdi = kwargs.get("apiqdi", "queryOffLineUserDevelop")
self.text = "实时数据"
def search_start(self):
"""融合前查询媒资表"""
result={}
for i in self.code:
WS_MERGEDMEDIA = self._creat_sql(", c2code from WS_MERGEDMEDIA where c2code='%s'" % i)
log().debug(WS_MERGEDMEDIA)
mer = db_oracle(WS_MERGEDMEDIA)
result[i] = {}
result[i] = {self.keys[i]: mer[0][i] for i in range(0, len(self.keys)) if mer}
if not result[i]:
raise AssertionError("code %s not data in table WS_MERGEDMEDIA" % i)
log().debug("%s---start--- WS_MERGEDMEDIA %s" % (i, result))
exec("self.start=result" )
def order_mvp(self):
"""订购总览，订购冠军"""
patten = '//*[@id="%s"]//tbody/tr' % self.table_id
self.wait_element('%s[1]' % patten)
eletrs_table = self.driver.find_elements_by_xpath(patten)
log().debug("{1} {0} content == {2}".format(self.table_id, patten, eletrs_table))
name = self.buttontext
self.screenhost_to_report(alt=name, name=name)
if not eletrs_table:
log().error("表 {0} 无数据 end".format(self.table_id))
self.getsysereorlog("end")
raise AssertionError("表 {0} 无数据 end".format(self.table_id))
tr_text = []
for tr in range(len(eletrs_table)):
patten_tr='{0}[{1}]/td'.format(patten,tr+1)
eletds_table = self.driver.find_elements_by_xpath(patten_tr)
td_text=[]
for td in range(len(eletds_table)):
patten_td = '{0}[{1}]'.format(patten_tr, td + 1)
log().debug(("tds", patten_td))
text_0=self.driver.find_element_by_xpath(patten_td).text
text_0=re.compile(r":|元\(金额\)|null").sub("",text_0)
td_text.append(text_0)
tr_text.append(td_text)
self.web_alldatt = tr_text
log().info("{0} content \n {1}".format(self.table_id, tr_text))
return tr_text
def search_stop(self):
"""
融合后查询数据库,生成字典
:return:
"""
self.toals = {}
for i in self.code:
if self.id_code.get(i, False):
self.code_element = i
WS_OUTSIDE = self._creat_sql(
", id from WS_OUTSIDE where id='%s'" % self.id_code[i])
out = db_oracle(WS_OUTSIDE)
text = " %s %s WS_OUTSIDE is %s " % (self.status, i, out)
log().debug(text)
self.outside = {
self.keys[i]: out[0][i]
for i in range(0, len(self.keys))
}
log().debug("%s WS_OUTSIDE %s" % (i, self.outside))
WS_MERGEDMEDIA = self._creat_sql(
", c2code from WS_MERGEDMEDIA where c2code='%s'" % i)
mer = db_oracle(WS_MERGEDMEDIA)
self.stop = {
self.keys[i]: mer[0][i]
for i in range(0, len(self.keys))
}
log().debug("%s WS_MERGEDMEDIA %s" % (i, self.stop))
self._data()
self.toals[i] = self.toal
log().info(self.toals)
def getc3_wait(self,timeout=10):
"""等待爬取完毕，每条等待时间为 timeout*3"""
strcode = str(self.code).replace("[", "").replace("]", "")
number = 0
sql = "select count(*) FROM WS_PROCESSDETAIL where SOURCE='3' and C2CODE in ({0})".format(strcode)
while True:
rescount = db_oracle(sql)[0][0]
log().debug("get over is %s" % rescount)
if rescount >= len(self.code):
break
elif number == timeout*3*len(self.code):
raise AssertionError("超时未获取C3数据 测试结束")
number += 1
time.sleep(1)
def checktext(self, num=1, element=""):
"""校验文本在页面是否存在"""
log().debug("校验文本在页面是否存在")
now = True
for i in range(0, bt.wait_time+5):
page = self.driver.page_source
if element in page:
now = False
break
time.sleep(1)
if now:
assert False, "\"{0}\" 不在当前页面".format(element)
elif not now and num == 1:
log().debug(" '%s' 在当前页，环境校验成功，next，" % element)
def merge_wait(self , timeout=20):
strcode = str(self.code).replace("[", "").replace("]", "")
number=0
sql = "SELECT count(*) from WS_PROCESS where PROCESSRESULT='2' and CODE in ({0})".format(strcode)
log().debug(sql)
while True:
rescount=db_oracle(sql)[0][0]
log().debug("merge over is %s" % rescount)
if rescount == len(self.code):
break
elif number == timeout:
raise AssertionError("timeout = %s 超时未融合完毕 测试结束" % timeout)
number += 1
time.sleep(1)
