def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="doLogin"]', } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 将cookie设置给session print(cookie_to_dict(cookie)) session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url url = f"http://hlqb.k39120.cn/customer/channelUser/getData?page=1&limit=10&channelId=&customerProvince=&createTimeStart={self.today}&createTimeEnd={self.today}&infoLevel=" # 请求url response = session.get(url, headers=headers) # 构造Selector info = response.json()['count'] print(info) # 获取结果 result = { "注册人数": info, "实名人数": "null", "申请人数": "null", "放款人数": "null", '备注': '' } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="form"]/div[3]/input', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = f"https://api.fzxfenqi.com/dkcsdrainage/getInfoDrainage?pageSize=10&pageNo=1&channel=mlh&beginTime={self.today}&endTime={self.today}&channelsCode=2&_={time.time()*1000}" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 过滤警告信息 warnings.filterwarnings("ignore") # 访问url response = session.get(url, headers=headers, verify=False) json_info = response.json()["result"][0] result = { "注册人数": json_info["registUsersNumber"], "实名人数": "null", "申请人数": json_info["applyLoanNumber"], "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="main-content"]/div/div/form/div[2]/button/span[2]', "check_code": '//*[@id="itemBox"]/div[3]/input', "code_image_url": '//*[@id="itemBox"]/div[4]/img', "success_ele": '/html/body/div[1]/div/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (336, 373, 627, 458), "30500") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://pxhadmin.wx273.com/Business/Promote/my_data?start={self.today}&end={self.today}&youxiao=" # 请求url response = session.get(page_url, headers=headers) # 构造Selector html = Selector(text=response.text) # 获取结果 result = { "注册人数": html.xpath('//*[@id="main"]/div[4]/div/span[2]/text()').re('共\s*(\d+)\s*条记录')[0], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="btn_login"]', } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url url = f"http://www.chaorenqianbao.com/qc/indexdata/?page=1&limit=20&mobile=&time_reg={self.today}+-+{self.today}&time_active=" # 请求url response = session.get(url, headers=headers) # 构造Selector info = response.json() # 获取结果 result = { "注册人数": info['count'], "实名人数": "null", "申请人数": "null", "放款人数": "null", '备注': '' } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div/div/div/div[2]/button', } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url url = f"https://saas.fin-tech.cn/admin/index.php/linkshare/index/share_count_ajax.html?page=1&limit=10&search_date_start={self.today}&search_date_end={self.today}" # 请求url response = session.get(url, headers=headers) # 构造Selector print(response.json()) info = response.json()['data']['0'] zc = re.sub("\D", "", info["count"]) # 获取结果 result = { "注册人数": zc, "实名人数": "null", "申请人数": info['submit_count'], "放款人数": "null", '备注': '' } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="login-form"]/div[5]/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = f"https://www.gglcqm.cn/admin/channel/counts/ids/83?addtabs=1" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(url, headers=headers) selector = Selector(text=response.text) apply = selector.xpath('//tbody/tr/td[3]/text()').extract()[0] print(apply) result = { "注册人数": re.sub('人', '', apply), "实名人数": "null", "申请人数": 'null', "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="sub"]', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) print(cookie_to_dict(cookie)) # json的url url = "http://de-hxr.91yiyongbao.com/selectClientTime" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } arg = { 'page': 1, 'statetime': f"{self.today} 00:00:00", 'endtime': f"{self.tomorrow} 00:00:00", } # 访问url response = session.post(url, headers=headers, data=arg) json_info = response.json() print(json_info) result = { "注册人数": json_info["linenumber"], "实名人数": "null", "申请人数": "null", "放款人数": "null", "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div[2]/div/form/div[4]/button', "check_code": '//*[@id="checkCode"]', "code_image_url": '//*[@id="check_code"]', "success_ele": '//*[@id="topbar-collapse"]/ul/li[1]/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (345, 365, 439, 403), "10400") # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # page的url page_url = "https://tg.shhlwlkj.com/customer/list.htm" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } arg = { 'mobile': "", 'status': "", 'brokerId': "", 'gmtCreateStart': f"{self.today}", 'gmtCreateEnd': f"{self.today}", 'currentPage': 1 } # 访问url response = session.post(page_url, headers=headers, data=arg) # 获取html info = Selector(text=response.text) print(response.text) # 最终结果 result = { "注册人数": info.xpath( '/html/body/div[1]/div[2]/div/div[3]/div/div/div/div/p/span/text()' ).extract()[0], "实名人数": "null", "申请人数": 'null', "放款人数": 'null', "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="main-content"]/div/div/form/div[2]/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) print(cookie_to_dict(cookie)) # json的url url = f"http://qd.jienihua100.com/Admin/Ditchs/indexAjax.html?start={self.today}&end={self.today}&url={self.channel}&sEcho=3&iColumns=3&sColumns=%2C%2C&iDisplayStart=0&iDisplayLength=10&mDataProp_0=date_text&sSearch_0=&bRegex_0=false&bSearchable_0=true&mDataProp_1=gl_nreg&sSearch_1=&bRegex_1=false&bSearchable_1=true&mDataProp_2=gl_login&sSearch_2=&bRegex_2=false&bSearchable_2=true&sSearch=&bRegex=false&_={str(int(time.time()*1000))}" #{str(int(time.time()*1000))} # url = f"http://qd.jienihua100.com/Admin/Ditchs/indexAjax.html?start=2019-01-09&end=2019-01-09&url=wyx03&sEcho=13&iColumns=3&sColumns=%2C%2C&iDisplayStart=0&iDisplayLength=10&mDataProp_0=date_text&sSearch_0=&bRegex_0=false&bSearchable_0=true&mDataProp_1=gl_nreg&sSearch_1=&bRegex_1=false&bSearchable_1=true&mDataProp_2=gl_login&sSearch_2=&bRegex_2=false&bSearchable_2=true&sSearch=&bRegex=false&_=1547018390672" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(url, headers=headers) json_info = response.json()['data'][0] print(json_info) result = { "注册人数": json_info["gl_nreg"], "实名人数": "null", "申请人数": "null", "放款人数": "null", "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="login"]', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) print(cookie_to_dict(cookie)) # json的url url = "http://demand.lianfen360.com/market?id=mk92rXg2&login=true" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(url, headers=headers) info = Selector(text=response.text) index = info.xpath('/html/body/div/div[6]/div/h2/text()').extract()[0] print(index) result = { "注册人数": re.search("注册数:(.+?);", index).group(1), "实名人数": "null", "申请人数": re.search("申请数:(.+?);", index).group(1), "放款人数": "null", "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="loginForm"]/div[4]/div/input', "check_code": '//*[@id="captcha"]', "code_image_url": '//*[@id="captchaBtn"]', "success_ele": '//*[@id="menu"]/li/ul/li/a/span[2]' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (300, 376, 600, 405), "30400") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) print(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://open-admin.zhongguoqiumi.com/channel/getChannelStatistics?channelName=&channelCode=mimkd3&startDate=2018-12-28&endDate=2018-12-28&numPerPage=10&pageNum=1&xzOldPageNum=1" # 请求url response = session.get(page_url, headers=headers) # 构造Selector selector = Selector(text=response.text) # 获取数据 info = selector.xpath('//*[@id="bigDataList"]/tbody/tr/td/text()').extract() # print(info) # 获取结果 result = { "注册人数": info[4], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="root"]/div/div/div[2]/div/form/div[4]/div/div/span/button', "check_code": '//*[@id="imgCode"]', "code_image_url": '//*[@id="root"]/div/div/div[2]/div/form/div[3]/div/div/span/div/div[2]/img', "success_ele": '//*[@id="root"]/div/div/div[1]/div/div/span/span' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (455, 315, 578, 355), "30400") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) print(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"https://www.souwl.cn/admin/register/index.shtml?start_time={self.today}+00%3A00%3A00&end_time={self.tomorrow}+00%3A00%3A00&channel=337" # 请求url response = session.get(page_url, headers=headers) # 构造Selector selector = Selector(text=response.text) # 获取数据 info = selector.xpath('/html/body/table[1]/tbody/tr/td/text()').extract() # 获取结果 result = { "注册人数": info[0], "实名人数": "null", "申请人数": "null", "放款人数": info[1] } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="root"]/div/div/form/div[3]/div/div/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = "http://c.ypuzhen.com/web/clUser/selectListAll" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } form = { 'pageNum': 1, 'pageSize': 2000, 'channelId': 284, 'startTime': f'{self.today}', 'endTime': f'{self.today}', } # 访问url response = session.post(url, headers=headers, data=form) json_info = response.json()['data']['pageDto'] #计算实名人数 num = json_info['list'] real_name = [] for i in num: name = i['realName'] if name: real_name.append(name) result = { "注册人数": json_info["total"], "实名人数": len(real_name), "申请人数": "null", "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="root"]/div/div/form/div[3]/div/div/button', "check_code": '', "code_image_url": '', "success_ele": '' } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # page的url page_url = "http://xfshchannel.tgjrfw.com/web/clUser/selectListAll" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36", } form = { "pageNum": "1", "pageSize": "2000", "channelId": "396", "startTime": f"{self.today}", "endTime": f"{self.today}" } # 访问url response = session.post(page_url, headers=headers, data=form) info = response.json()["data"]["pageDto"] # 最终结果 result = { "注册人数": info['total'], "实名人数": 0, "申请人数": 'null', "放款人数": 'null', "备注": "" } sm = info["list"] for each in sm: # print(each) if each["realName"]: result["实名人数"] += 1 # print(result) self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="login-page-full"]/div/div[3]/form/ul/li[4]/button', "check_code": '//*[@id="login-page-full"]/div/div[3]/form/ul/li[3]/input', "code_image_url": '//*[@id="login-page-full"]/div/div[3]/form/ul/li[3]/span/img', "success_ele": '//*[@id="header-navbar"]/div/div[1]/ul/li/a/img' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (616, 405, 710, 438), "20300") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://0v10.cn/admin/customer/promote.html?start_time={self.today.year}%2F{str(100+int(self.today.month))[1:]}%2F{str(100+self.today.day)[1:]}+-+{self.tomorrow.year}%2F{str(100+int(self.tomorrow.month))[1:]}%2F{str(100+self.tomorrow.day)[1:]}" print(page_url) # 请求url response = session.get(page_url, headers=headers) # 构造Selector selector = Selector(text=response.text) # 获取数据 info = selector.xpath( '//*[@id="content-wrapper"]/div/div/div[2]/div/div/div/div[2]/form/table//tr/td[4]/text()' ).extract()[0] print(info) # 获取结果 result = { "注册人数": info, "实名人数": "null", "申请人数": "null", "放款人数": "null", '备注': '' } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="login-form"]/div/button', "check_code": '', "code_image_url": '', "success_ele": '' } # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 将cookie设置给session session = Session() # 设置session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url url = f"https://ssda.xyshuj.com/?start_date={self.today}&end_date={self.today}" # 请求url response = session.get(url, headers=headers) # 获取结果 html = Selector(text=response.text) result = { "注册人数": html.xpath( '//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[3]/text()' ).extract()[0], "实名人数": "null", "申请人数": html.xpath( '//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[4]/text()' ).extract()[0], "放款人数": html.xpath( '//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[5]/text()' ).extract[0], "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="root"]/div/div/form/div[3]/div/div/span/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) print(cookie) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = "http://c.wedom.cn/web/clUser/selectUserCount" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } arg = { 'channelId': "528", 'endTime': f"{self.today}", 'pageNum': 1, 'pageSize': 20, 'startTime': f"{self.today}", } # 访问url response = session.post(url, headers=headers, json=arg) info = response.json()['data'][0] print(info) result = { "注册人数": info['countUser'], "实名人数": "null", "申请人数": info['countBorrow'], "放款人数": info['countSuccessBorrow'], "备注": '' } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="login"]/form/button', "check_code": '', "code_image_url": '', "success_ele": '' } # 设置session session = Session() # 获取cookie cookie = cookie_to_dict(self.no_check_get_cookie(xpath_info)) # cookie["pagenum"] = "200" # 将cookie设置给session session.cookies.update(cookie) # print(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url json_url = "https://toupin.cn/qianbaoadm/channel/list" # 请求url #post arg = { 'addtime_endtime': f'{self.today}', 'addtime_starttime': f'{self.tomorrow}', 'currentpage': "1" } response = session.post(json_url, headers=headers, json=json.dumps(arg)) # print(response.text) # print(response.status_code) info = response.json()['data'] # 获取结果 result = { "注册人数": info['listnum'], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="logina"]', "check_code": '', "code_image_url": '', "success_ele": '' } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = "http://haoshun.youdaikeji.com/mgmt/thirdChannel/channelStatistics?channelId=94" #参数 arg = { 'startDateTime': f'{self.today}', # 'channelId':, 'page': 1, 'rows': 10, } # 请求url response = session.post(page_url, headers=headers, data=arg) # 构造Selector data = response.json()['rows'][0] print(data) # 获取结果 result = { "注册人数": data['countUser'], "实名人数": 'null', "申请人数": data['countApply'], "放款人数": "null" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div/form/input[4]', "check_code": '/html/body/div/form/input[3]', "code_image_url": '/html/body/div/form/img', "success_ele": '/html/body/div[1]/ul/li[2]/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (502, 390, 638, 439), "10400") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = "http://www.lovezcm.com/ymd/admin/channel-view.jsp?id=0.7025606997962637" # 请求url response = session.get(page_url, headers=headers) # 构造Selector html = Selector(text=response.text) # 获取结果 result = { "注册人数": html.xpath('/html/body/div[1]/div[2]/table//tr[2]/td[3]/text()'). extract()[0], "实名人数": "null", "申请人数": "null", "放款人数": "null", "备注": "" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="ju"]/form/button', "check_code": '//*[@id="ju"]/form/div[3]/input', "code_image_url": '//*[@id="c2c98f0de5a04167a9e427d883690ff6"]', "success_ele": '/html/body/div/header/a/span[2]' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (467, 569, 619, 621), "30500") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://openback.hongzigame.com/channel/getChannelStatistics?channelName=&channelCode={self.channel}&startDate={self.today}&endDate=&numPerPage=10&pageNum=1&xzOldPageNum=1" # 请求url response = session.get(page_url, headers=headers) # 构造Selector selector = Selector(text=response.text) # 获取数据 info = selector.xpath( '//*[@id="bigDataList"]/tbody/tr/td/text()').extract() print(info) # 获取结果 result = { "注册人数": info[4], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="loginBtn"]', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = 'http://channel.mgshujia.com/api?cmd=chluserTotal' # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } #postn内容 arg = { 'cnds': f'%7B%22date_FROM_cnd%22%3A%22{self.today}%22%2C%22date_TO_cnd%22%3A%22{self.today}%22%2C%22channel_path_cnd%22%3A%22%22%2C%22is_fuzzy_cnd%22%3A%22%22%7D' } # 访问url response = session.post(url, headers=headers, data=arg) # print(response.status_code) # print(response.json()) json_info = response.json()["rows"] result = { "注册人数": json_info["register_num"], "实名人数": json_info['complete_info_num'], "申请人数": 'null', "放款人数": json_info['credit_pass_num'] } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="form"]/div[4]/div/a[1]', "check_code": '//*[@id="form"]/div[3]/div/input', "code_image_url": '//*[@id="verify_img"]', "success_ele": '//*[@id="推广产品"]/span' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (984, 454, 1148, 503), "10400") # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # page的url page_url = f"http://s.51jinkong.cn/admin/channelmenu/index?shop_id=86&the_year_month={self.today.year}-{self.today.month}" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(page_url, headers=headers) # 获取html html = response.text # 构造选择器 selector = Selector(text=html) # 获取数据 info = selector.xpath('/html/body/div[4]/div/article/table[2]/tbody/tr[1]') registers = info.xpath('./td[2]/text()').extract()[0].strip() apply = info.xpath('./td[3]/font/text()').extract()[0] success = info.xpath('.//td[7]/font/text()').extract()[0] # 最终结果 result = { "注册人数": registers, "实名人数": "null", "申请人数": apply, "放款人数": success } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="loginForm"]/div[5]/div/input', "check_code": '//*[@id="captcha"]', "code_image_url": '//*[@id="loginForm"]/div[4]/div[2]/img', "success_ele": '//*[@id="menu"]/li/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (300, 376, 600, 405), "30400") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = 'http://back.yzb668.com/diversion/statistics' # 请求url response = session.get(page_url, headers=headers) # 构造Selector selector = Selector(text=response.text) # 获取数据 # print(selector) info = selector.xpath( '//*[@id="bigDataList"]/tbody/tr/td/text()').extract() # 获取结果 result = { "注册人数": info[2], "实名人数": info[3], "申请人数": "null", "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div/div/div/form/div[4]', "check_code": '//*[@id="captcha"]', "code_image_url": '//*[@id="validateCodeImg"]', "success_ele": '/html/body/div/div[1]/div[2]/ul' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (507, 487, 638, 530), "10400") # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # page的url page_url = "http://partner.ytrong.com/partner/info?id=310&page=1&limit=30" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } arg = {'id': 310, 'page': 1, 'limit': 30} # 访问url response = session.post(page_url, headers=headers, data=arg) # 获取html info = response.json()['rows'][0] # print(info) # 最终结果 result = { "注册人数": info['userCount'], "实名人数": "null", "申请人数": 'null', "放款人数": 'null', "备注": "" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="app"]/div/form/div[3]/div/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url json_url = f"http://crm.channel.51zhihe.com/backend/statistics?page=1&data=%7B%22search%22:%7B%22start%22:%22{self.today}%22,%22end%22:%22{self.today}%22,%22name%22:%22%22%7D%7D" # 设置头部 headers = { "Host": f"{cookie[0]['domain']}", "Referer": f"http://{cookie[0]['domain']}/backend/index", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(json_url, headers=headers) # 获取数据 json_info = response.json()["data"]["lists"]["data"][0] # 最终结果 result = { "注册人数": json_info["registers"], "实名人数": "null", "申请人数": json_info["applications"], "放款人数": "null" } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="rrapp"]/div[2]/div[4]/div[2]/button', "check_code": '//*[@id="rrapp"]/div[2]/div[3]/input', "code_image_url": '//*[@id="rrapp"]/div[2]/div[3]/img', "success_ele": '//*[@id="rrapp"]/header/nav/div[1]' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (525, 305, 633, 340), "30500") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://106.14.158.253:8090//getRegisterCount?_search=false&nd={str(int(time.time()*1000))}&size=10¤t=1&sidx=&order=asc&startTime={self.today}&endTime={self.today}&channelCode=ka10&_={str(int(time.time()*1000))}" # 请求url response = session.get(page_url, headers=headers) # 构造Selector data = response.json()['records'][0] print(data) # 获取结果 result = { "注册人数": data['registerCount'], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div/div/div/form/div[4]', "check_code": '//*[@id="captcha"]', "code_image_url": '//*[@id="validateCodeImg"]', "success_ele": '/html/body/div/div[1]/div[2]/ul/li[1]/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (504, 492, 635, 535), "10400") # 将cookie设置给session session.cookies.update(cookie_to_dict(cookie)) # 设置头部信息 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 页面url page_url = f"http://partner.ytrong.com/partner/info?id=354&page=1&limit=30" # 请求url response = session.get(page_url, headers=headers) # 构造Selector data = response.json()['rows'][0] print(data) # 获取结果 result = { "注册人数": data['userCount'], "实名人数": "null", "申请人数": "null", "放款人数": "null" } self.write_sql(result) print(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '/html/body/div/div[2]/form/div[3]/div[2]/button', "check_code": "", "code_image_url": "", "success_ele": "" } # 设置session session = Session() # 获取cookie cookie = self.no_check_get_cookie(xpath_info) print(cookie) # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # json的url url = f"http://report.xdy7.com/user/leads/daily?source={self.channel}&datebegin={self.today}&dateend={self.today}" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(url, headers=headers) info = Selector(text=response.text) result = { "注册人数": info.xpath('//table//tr/td[2]/text()').extract()[0], "实名人数": "null", "申请人数": "null", "放款人数": "null", "备注": '' } self.write_sql(result)
def get_info(self): xpath_info = { "username": '******', "password": '******', "login_button": '//*[@id="loginForm"]/div[5]/button', "check_code": '//*[@id="verify"]', "code_image_url": '//*[@id="verifyImg"]', "success_ele": '//*[@id="navbar-header"]/ul[1]/li[3]/a' } # 设置session session = Session() # 获取cookie cookie = self.check_get_cookie(xpath_info, (563, 486, 656, 522), "30400") # 给session设置cookie session.cookies.update(cookie_to_dict(cookie)) # page的url page_url = "https://quan.jsd0086.com/agent/report/index" # 设置头部 headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36", } # 访问url response = session.get(page_url, headers=headers) # 获取html info = Selector(text=response.text) print(response.text) # 最终结果 result = { "注册人数": info.xpath('//table//tr/td[2]/text()').extract()[0], "实名人数": info.xpath('//table//tr/td[5]/text()').extract()[0], "申请人数": 'null', "放款人数": 'null', "备注": "" } self.write_sql(result)