def request_data_home(self): """构造首页请求数据""" start_time = self.start_time end_time = self.end_time self.data = \ { "qdi": self.apiqdi, "starttime": start_time, "begintime": start_time, 'begindate': start_time, "endtime": end_time, "enddate": end_time, "datetype": self.apidatatype, "rootcategoryid": None, "groupcode": getattr(self, "groupcode", None), "hours": "00:00-00:59", "token": self.token, "tagid":None, "vspid":"1", "selectdate": getattr(self, "request_code", None), "channelcode":getattr(self, "request_code", None), "sysid": getattr(self,"secsiono","u"), "userid": "test1", } if getattr(self, "movetype","") == "全网频道": self.data["groupcode"] = None log().debug("requests 'url, data' \t {1},{0}".format(self.data, self.url_api))
def api_result(self): """发送请求并返回数据""" headr = { "Accept": "application/json, text/javascript, */*; q=0.01", "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", } try: res = requests.post(self.url_api, data=self.data, headers=headr, timeout=bt.wait_ele-4) text = eval(res.text) except NameError: """str>>dict""" text = json.loads(res.text) except ReadTimeout as e: """Timeout""" self.e=e text = dict() self.datatext = list() log().error(e) self.datatext=text.get("subject", [])[0:2] log().info("result subject data is {1}\t ".format(text, self.datatext)) if self.datatext: Ioput.input("datatext", self.datatext) else: raise AssertionError(r"statplatform5 output not data ,test stop \n[error :{0}] \n request data is: {1}".format(getattr(self, "e", None), self.data))
def keyword_search(self, ): """关键字搜索""" data = self.table_firstline_list name1 = [i[0] for i in data] number = 0 while True: """随机关键字""" randomnamepice = random.choice(name1) sindex = random.randint(0, len(randomnamepice)) eindex = random.randint(0, len(randomnamepice)) if sindex < eindex: keyword = randomnamepice[sindex:eindex] break elif number >= 100: """100次不合法 置空""" keyword = "" break number += 1 log().debug("keyword is {0}".format(keyword)) kw_ele_id = "%sSearch" % self.table_id self.exement(elements=kw_ele_id, ele_type="id", parms=keyword) data = self._get_tabledata(tablenum="2", getlines=20)[1] log().debug("after search keywords {0}".format(data)) self.screenhost_to_report(alt="关键字搜索", name="关键字搜索") for i in data: if keyword not in i[0]: raise AssertionError("关键字 : \"%s\" %s 在查询的结果中不存在" % (keyword, i))
def set_rule(self): """设置待匹配的字段,规则""" for i in self.rule: sql_init = "INSERT INTO WS_RULE VALUES('%d','%s','%s','%s')" % (self.rule.index(i), i[0], i[0], i[1]) log().debug(sql_init) db_oracle(sql_init)
def move_detaildata(self): """上下移动,截屏""" log().debug("滚动到屏幕顶部") """滚动到屏幕底部获取详情数据,并截屏""" scroll_top_js = "window.scrollTo(0,0);" #顶端 scroll_end_js = "window.scrollTo(0,document.body.scrollHeight);" #底部 scroll_view_js = "arguments[0].scrollIntoView();" #指定元素 time.sleep(0.2) ele_top = "" classname = self.__class__.__name__ if hasattr(self, "text") and self.text == "查询": ele_top = self.driver.find_elements_by_id("searchButton") elif hasattr(self, "p") and self.p == "groupToggle": ele_top = self.driver.find_elements_by_id("groupToggle") elif hasattr(self, "up_text"): ele_top = self.driver.find_elements(**self.up_text) elif "详情" in classname: try: ele_top = self.driver.find_elements_by_id("nameTitle") except: ele_top = self.driver.find_elements_by_id("select2-changetime-container") if ele_top: self.driver.execute_script(scroll_view_js, ele_top[0]) # 移动到 顶端 name = os.path.splitext(self.dlfile)[0]+"表" self.screenhost_to_report(alt=name+"header", name=name+"header") log().debug("滚动到屏幕底部") eledown = self.driver.find_elements(**self.down_text) if eledown: self.driver.execute_script(scroll_view_js, eledown[0]) #移动到指定元素 底部
def _set_time_day(self): """日报""" log().debug("日报") try: self.exement("day", "id") except AssertionError as a: if "分钟收视" not in self.__class__.__name__ : raise AssertionError(a) starttime = self.starttime endtime = self.endtime sy = starttime.split("-")[0] sm = str(int(starttime.split("-")[1]) - 1) ey = endtime.split("-")[0] em = str(int(endtime.split("-")[1]) - 1) self.exement("starttime","id") self.exement('//*[@class="layui-laydate"]//*[@lay-type="year"]') self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % sy) self.exement('//*[@class="layui-laydate"]//*[@lay-type="month"]') self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % sm) self.exement('//*[@class="layui-laydate"]//*[@lay-ymd="%s"]' % starttime) self.exement("endtime", "id") self.exement('//*[@class="layui-laydate"]//*[@lay-type="year"]') self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % ey) self.exement('//*[@class="layui-laydate"]//*[@lay-type="month"]') self.exement('//*[@class="layui-laydate"]//*[@lay-ym="%s"]' % em) self.exement('//*[@class="layui-laydate"]//*[@lay-ymd="%s"]' % endtime)
def activity(self, s): """更改ws_process表,触发融合操作""" for i in self.code: sql="update WS_PROCESS set PROCESSRESULT='%s' where code='%s'" % (s,i) db_oracle(sql) log().debug([i, db_oracle("select PROCESSRESULT from WS_PROCESS where code='%s' " % i)]) self.status = "stop"
def get_tablefirstdata(self): """获取详情数据的第一行""" log().debug(self.table_id) self.wait_element(self.table_id, "id") self.ele_Table = self.exement(self.table_id, "id", "getele") #self.driver.find_element_by_id(self.table_id) self.ele_tables = self.ele_Table.find_elements_by_xpath('//div[starts-with(@class,"table-")]') self.data_lines, self.table_firstline_list = self._get_tabledata("2") log().info("table content is %s" % self.table_firstline_list)
def count_user(self): """""" self.wait_element(self.table_id, "id") element = self.driver.find_element_by_id(self.table_id) text = element.text if text == "0" or text == "0%": log().error("表 {0} 无数据 end".format(self.table_id)) self.getsysereorlog("end") raise AssertionError("表 {0} 无数据 end".format(self.table_id))
def init(self): """初始化环境""" strcode = str(self.id_list).replace("[", "").replace("]", "") clean_side= "delete WS_OUTSIDE where id in ({0})".format(strcode) db_oracle(clean_side) clean_tail = "delete WS_PROCESSDETAIL where id in ({0})".format(strcode) db_oracle(clean_tail) log().info("clean WS_PROCESSDETAIL and WS_OUTSIDE sucess")
def _set_time_week(self): """月报""" log().debug("周报") self.exement('//*[@id="week"]') self.exement('//*[@id="select2-starttime-container"]') self.exement('//*[@id="select2-starttime-results"]//li[text()="{0}"]'.format(self.starttime)) self.exement('//*[@id="select2-endtime-container"]') self.exement('//*[@id="select2-endtime-results"]//li[text()="{0}"]'.format(self.endtime))
def order_top5(self): """top5""" patten='//*[@id="{0}"]/div[1]/canvas'.format(self.table_id) self.wait_element('%s[1]' % patten) elements = self.driver.find_elements_by_xpath('//*[@id="{0}"]/div[1]/canvas'.format(self.table_id)) if not elements: log().error("表 {0} 无数据 end".format(self.table_id)) self.getsysereorlog("end") raise AssertionError("表 {0} 无数据 end".format(self.table_id))
def open(self): """打开浏览器""" options = webdriver.ChromeOptions() prefs = {'profile.default_content_settings.popups': 0, 'download.default_directory': BI_protal.downloadpath} options.add_experimental_option('prefs', prefs) options.add_argument('log-level=3') self.driver = webdriver.Chrome(executable_path=BI_protal.driver_path,chrome_options=options) self.driver.maximize_window() self.driver.implicitly_wait(BI_protal.wait_time) log().info("{0}启动浏览器{0}".format("*" * 40))
def wait_element(self, elementstr, element_type="xpath", seq=0.5): """显示等待""" self.elementstr = elementstr self.element_type = element_type log().info("轮询元素 %s : %s " % (self.element_type, self.elementstr)) try: WebDriverWait(self.driver, self.alltime, seq).until(self) except: log().error("超时,timeout >= %s秒 元素 <%s : %s> 未加载;" % (self.alltime, element_type, elementstr)) raise AssertionError("超时,timeout >= %s 元素 <%s : %s> 未加载;" % (self.alltime, element_type, elementstr))
def test_c2c3(self): """未存在C3数据时,爬取扩展数据+融合数据""" I.function_name(self.__class__.__name__) try: obj = GetC3Data(**{'rule': [('YEAR', '1')]}) obj.execute_c2c3() except AssertionError as e: log().error(e) self.assertTrue("") else: self.assertEqual(obj.status, "pass")
def look(self): """点击第一条的查看,切入详情页""" log().debug("点击‘查看’,进入详情页") """点击查看,并切换到frame""" try: self.ele_Table.find_elements_by_xpath('//a[text()="查看"]')[0].click() except: log().error("详情数据表中,无数据") raise AssertionError("无数据") self.driver.switch_to.parent_frame() self.frame_change(-1)
def executeapi(self, url, data={}): self.replacepar(url=url) if data: self.result = requests.post(url=self.url, data=data, timeout=self.timeout) elif not data: self.result = requests.get(url=self.url, timeout=self.timeout) log().info("\n{1}{0}".format(self.result.text, "\t" * 2)) log().info(self.result.url)
def get_text(self): e = [] try: try: self.text = eval(self.result.text) except Exception as e1: e.append(e1) self.text = json.loads(self.result.text) except Exception as e2: e.append(e2) log().error("不合发的数据") raise Exception(e)
def __searchsql(self): """根据qdi匹配日志中的sql""" lastlist = self.alllist[ListenServerLog.Numbers:] strs = "".join(lastlist) p1 = r"\d.+?%s:.+?(\[oracle.+?\])\n+" % self.qdi p2 = r"\d.+?%s:.+?Preparing: (select.+?)\n+" % self.qdi p3 = r"\d.+?%s:.+?Parameters: ((?:.+\(.+?\), ){0,}(?:.+\(.+?\)))\n" % self.qdi pattern = p1 + p2 + p3 self.list = re.compile(pattern=pattern).findall(strs) log().info("%s search count %s = %s" % (self.status, self.qdi, len(self.list)))
def test_exitsc3(self): """已有C3数据时爬去扩展数据""" I.function_name(self.__class__.__name__) try: obj = GetC3Data() obj.execute_exitsc3() except AssertionError as e: log().error(e) self.assertTrue("") else: self.assertEqual(obj.status, "pass")
def test_idpc(self, kwargs): """测试融合规则""" I.function_name(self.__class__.__name__, getdata("data.txt", "idpc").index(kwargs) + 1) try: obj = Medial(**kwargs) obj.execute_c2() except AssertionError as e: log().error(e) self.assertTrue("", e) else: self.assertEqual(obj.status, "pass")
def search_c3(self): """查询C3数据""" strcode = str(self.code).replace("[", "").replace("]", "") sql_id = "select ID FROM WS_PROCESSDETAIL where SOURCE='3' and C2CODE in ({0})".format(strcode) resid = db_oracle(sql_id) id_list=[i[0] for i in resid] strid_list = str(id_list).replace("[", "").replace("]", "") sql_outside="SELECT count(*) from WS_OUTSIDE where ID in ({0})".format(strid_list) res_side=db_oracle(sql=sql_outside) self.id_list=id_list self.side=res_side[0][0] log().info("%s WS_PROCESSDETAIL ID have %s, WS_OUTSIDE ID have %s " % (self.status ,self.id_list, self.side))
def change_sec(self): """电信、移动、联通节点选择""" self.wait_element('//*[@id="div-select"]/form/span/label', "xpath") sec_text = self.driver.find_element_by_xpath('//*[@id="div-select"]/form/span/label').text sec_text = sec_text.strip() log().debug((sec_text, type(sec_text))) d = {"中国电信": "1", "中国移动": "2", "中国联通": "3"} if d[sec_text] != self.sec_text: self.exement('//*[@id="div-select"]/form/span') time.sleep(0.5) self.exement('//*[@id="div-select"]/form/ul/li[%s]/a' % self.sec_text) time.sleep(0.5) log().info("landing %s sucess , secaction %s" % (self.url, self.sec_text))
def pararms(self, **kwargs): """设置参数""" log().debug("%s" % kwargs) self.timetype = kwargs.get("timetype", self.time_type) self.starttime = self.Times[self.timetype].get("starttime", bt.starttime) self.endtime = self.Times[self.timetype].get("endtime", bt.endtime) self.apidatatype = self.Times[self.timetype].get("datatype","D") self.user_index = kwargs.get("user_index", "1") self.table_id = kwargs.get("table_id", "userDevelopTable") self.dlfile = kwargs.get("dlfile", "用户发展数据.xlsx") self.apiqdi = kwargs.get("apiqdi", "queryOffLineUserDevelop") self.text = "实时数据"
def search_start(self): """融合前查询媒资表""" result={} for i in self.code: WS_MERGEDMEDIA = self._creat_sql(", c2code from WS_MERGEDMEDIA where c2code='%s'" % i) log().debug(WS_MERGEDMEDIA) mer = db_oracle(WS_MERGEDMEDIA) result[i] = {} result[i] = {self.keys[i]: mer[0][i] for i in range(0, len(self.keys)) if mer} if not result[i]: raise AssertionError("code %s not data in table WS_MERGEDMEDIA" % i) log().debug("%s---start--- WS_MERGEDMEDIA %s" % (i, result)) exec("self.start=result" )
def order_mvp(self): """订购总览,订购冠军""" patten = '//*[@id="%s"]//tbody/tr' % self.table_id self.wait_element('%s[1]' % patten) eletrs_table = self.driver.find_elements_by_xpath(patten) log().debug("{1} {0} content == {2}".format(self.table_id, patten, eletrs_table)) name = self.buttontext self.screenhost_to_report(alt=name, name=name) if not eletrs_table: log().error("表 {0} 无数据 end".format(self.table_id)) self.getsysereorlog("end") raise AssertionError("表 {0} 无数据 end".format(self.table_id)) tr_text = [] for tr in range(len(eletrs_table)): patten_tr='{0}[{1}]/td'.format(patten,tr+1) eletds_table = self.driver.find_elements_by_xpath(patten_tr) td_text=[] for td in range(len(eletds_table)): patten_td = '{0}[{1}]'.format(patten_tr, td + 1) log().debug(("tds", patten_td)) text_0=self.driver.find_element_by_xpath(patten_td).text text_0=re.compile(r":|元\(金额\)|null").sub("",text_0) td_text.append(text_0) tr_text.append(td_text) self.web_alldatt = tr_text log().info("{0} content \n {1}".format(self.table_id, tr_text)) return tr_text
def search_stop(self): """ 融合后查询数据库,生成字典 :return: """ self.toals = {} for i in self.code: if self.id_code.get(i, False): self.code_element = i WS_OUTSIDE = self._creat_sql( ", id from WS_OUTSIDE where id='%s'" % self.id_code[i]) out = db_oracle(WS_OUTSIDE) text = " %s %s WS_OUTSIDE is %s " % (self.status, i, out) log().debug(text) self.outside = { self.keys[i]: out[0][i] for i in range(0, len(self.keys)) } log().debug("%s WS_OUTSIDE %s" % (i, self.outside)) WS_MERGEDMEDIA = self._creat_sql( ", c2code from WS_MERGEDMEDIA where c2code='%s'" % i) mer = db_oracle(WS_MERGEDMEDIA) self.stop = { self.keys[i]: mer[0][i] for i in range(0, len(self.keys)) } log().debug("%s WS_MERGEDMEDIA %s" % (i, self.stop)) self._data() self.toals[i] = self.toal log().info(self.toals)
def getc3_wait(self,timeout=10): """等待爬取完毕,每条等待时间为 timeout*3""" strcode = str(self.code).replace("[", "").replace("]", "") number = 0 sql = "select count(*) FROM WS_PROCESSDETAIL where SOURCE='3' and C2CODE in ({0})".format(strcode) while True: rescount = db_oracle(sql)[0][0] log().debug("get over is %s" % rescount) if rescount >= len(self.code): break elif number == timeout*3*len(self.code): raise AssertionError("超时未获取C3数据 测试结束") number += 1 time.sleep(1)
def checktext(self, num=1, element=""): """校验文本在页面是否存在""" log().debug("校验文本在页面是否存在") now = True for i in range(0, bt.wait_time+5): page = self.driver.page_source if element in page: now = False break time.sleep(1) if now: assert False, "\"{0}\" 不在当前页面".format(element) elif not now and num == 1: log().debug(" '%s' 在当前页,环境校验成功,next," % element)
def merge_wait(self , timeout=20): strcode = str(self.code).replace("[", "").replace("]", "") number=0 sql = "SELECT count(*) from WS_PROCESS where PROCESSRESULT='2' and CODE in ({0})".format(strcode) log().debug(sql) while True: rescount=db_oracle(sql)[0][0] log().debug("merge over is %s" % rescount) if rescount == len(self.code): break elif number == timeout: raise AssertionError("timeout = %s 超时未融合完毕 测试结束" % timeout) number += 1 time.sleep(1)