def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="doLogin"]',
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 将cookie设置给session
print(cookie_to_dict(cookie))
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
url = f"http://hlqb.k39120.cn/customer/channelUser/getData?page=1&limit=10&channelId=&customerProvince=&createTimeStart={self.today}&createTimeEnd={self.today}&infoLevel="
# 请求url
response = session.get(url, headers=headers)
# 构造Selector
info = response.json()['count']
print(info)
# 获取结果
result = {
"注册人数": info,
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
'备注': ''
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="form"]/div[3]/input',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = f"https://api.fzxfenqi.com/dkcsdrainage/getInfoDrainage?pageSize=10&pageNo=1&channel=mlh&beginTime={self.today}&endTime={self.today}&channelsCode=2&_={time.time()*1000}"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 过滤警告信息
warnings.filterwarnings("ignore")
# 访问url
response = session.get(url, headers=headers, verify=False)
json_info = response.json()["result"][0]
result = {
"注册人数": json_info["registUsersNumber"],
"实名人数": "null",
"申请人数": json_info["applyLoanNumber"],
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="main-content"]/div/div/form/div[2]/button/span[2]',
"check_code": '//*[@id="itemBox"]/div[3]/input',
"code_image_url": '//*[@id="itemBox"]/div[4]/img',
"success_ele": '/html/body/div[1]/div/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (336, 373, 627, 458), "30500")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://pxhadmin.wx273.com/Business/Promote/my_data?start={self.today}&end={self.today}&youxiao="
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
html = Selector(text=response.text)
# 获取结果
result = {
"注册人数": html.xpath('//*[@id="main"]/div[4]/div/span[2]/text()').re('共\s*(\d+)\s*条记录')[0],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="btn_login"]',
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
url = f"http://www.chaorenqianbao.com/qc/indexdata/?page=1&limit=20&mobile=&time_reg={self.today}+-+{self.today}&time_active="
# 请求url
response = session.get(url, headers=headers)
# 构造Selector
info = response.json()
# 获取结果
result = {
"注册人数": info['count'],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
'备注': ''
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div/div/div/div[2]/button',
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
url = f"https://saas.fin-tech.cn/admin/index.php/linkshare/index/share_count_ajax.html?page=1&limit=10&search_date_start={self.today}&search_date_end={self.today}"
# 请求url
response = session.get(url, headers=headers)
# 构造Selector
print(response.json())
info = response.json()['data']['0']
zc = re.sub("\D", "", info["count"])
# 获取结果
result = {
"注册人数": zc,
"实名人数": "null",
"申请人数": info['submit_count'],
"放款人数": "null",
'备注': ''
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="login-form"]/div[5]/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = f"https://www.gglcqm.cn/admin/channel/counts/ids/83?addtabs=1"
# 设置头部
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(url, headers=headers)
selector = Selector(text=response.text)
apply = selector.xpath('//tbody/tr/td[3]/text()').extract()[0]
print(apply)
result = {
"注册人数": re.sub('人', '', apply),
"实名人数": "null",
"申请人数": 'null',
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="sub"]',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
print(cookie_to_dict(cookie))
# json的url
url = "http://de-hxr.91yiyongbao.com/selectClientTime"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
arg = {
'page': 1,
'statetime': f"{self.today} 00:00:00",
'endtime': f"{self.tomorrow} 00:00:00",
}
# 访问url
response = session.post(url, headers=headers, data=arg)
json_info = response.json()
print(json_info)
result = {
"注册人数": json_info["linenumber"],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
"备注": ""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div[2]/div/form/div[4]/button',
"check_code": '//*[@id="checkCode"]',
"code_image_url": '//*[@id="check_code"]',
"success_ele": '//*[@id="topbar-collapse"]/ul/li[1]/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (345, 365, 439, 403),
"10400")
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# page的url
page_url = "https://tg.shhlwlkj.com/customer/list.htm"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
arg = {
'mobile': "",
'status': "",
'brokerId': "",
'gmtCreateStart': f"{self.today}",
'gmtCreateEnd': f"{self.today}",
'currentPage': 1
}
# 访问url
response = session.post(page_url, headers=headers, data=arg)
# 获取html
info = Selector(text=response.text)
print(response.text)
# 最终结果
result = {
"注册人数":
info.xpath(
'/html/body/div[1]/div[2]/div/div[3]/div/div/div/div/p/span/text()'
).extract()[0],
"实名人数":
"null",
"申请人数":
'null',
"放款人数":
'null',
"备注":
""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button":
'//*[@id="main-content"]/div/div/form/div[2]/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
print(cookie_to_dict(cookie))
# json的url
url = f"http://qd.jienihua100.com/Admin/Ditchs/indexAjax.html?start={self.today}&end={self.today}&url={self.channel}&sEcho=3&iColumns=3&sColumns=%2C%2C&iDisplayStart=0&iDisplayLength=10&mDataProp_0=date_text&sSearch_0=&bRegex_0=false&bSearchable_0=true&mDataProp_1=gl_nreg&sSearch_1=&bRegex_1=false&bSearchable_1=true&mDataProp_2=gl_login&sSearch_2=&bRegex_2=false&bSearchable_2=true&sSearch=&bRegex=false&_={str(int(time.time()*1000))}" #{str(int(time.time()*1000))}
# url = f"http://qd.jienihua100.com/Admin/Ditchs/indexAjax.html?start=2019-01-09&end=2019-01-09&url=wyx03&sEcho=13&iColumns=3&sColumns=%2C%2C&iDisplayStart=0&iDisplayLength=10&mDataProp_0=date_text&sSearch_0=&bRegex_0=false&bSearchable_0=true&mDataProp_1=gl_nreg&sSearch_1=&bRegex_1=false&bSearchable_1=true&mDataProp_2=gl_login&sSearch_2=&bRegex_2=false&bSearchable_2=true&sSearch=&bRegex=false&_=1547018390672"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(url, headers=headers)
json_info = response.json()['data'][0]
print(json_info)
result = {
"注册人数": json_info["gl_nreg"],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
"备注": ""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="login"]',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
print(cookie_to_dict(cookie))
# json的url
url = "http://demand.lianfen360.com/market?id=mk92rXg2&login=true"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(url, headers=headers)
info = Selector(text=response.text)
index = info.xpath('/html/body/div/div[6]/div/h2/text()').extract()[0]
print(index)
result = {
"注册人数": re.search("注册数:(.+?);", index).group(1),
"实名人数": "null",
"申请人数": re.search("申请数:(.+?);", index).group(1),
"放款人数": "null",
"备注": ""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="loginForm"]/div[4]/div/input',
"check_code": '//*[@id="captcha"]',
"code_image_url": '//*[@id="captchaBtn"]',
"success_ele": '//*[@id="menu"]/li/ul/li/a/span[2]'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (300, 376, 600, 405), "30400")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
print(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://open-admin.zhongguoqiumi.com/channel/getChannelStatistics?channelName=&channelCode=mimkd3&startDate=2018-12-28&endDate=2018-12-28&numPerPage=10&pageNum=1&xzOldPageNum=1"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
selector = Selector(text=response.text)
# 获取数据
info = selector.xpath('//*[@id="bigDataList"]/tbody/tr/td/text()').extract()
# print(info)
# 获取结果
result = {
"注册人数": info[4],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="root"]/div/div/div[2]/div/form/div[4]/div/div/span/button',
"check_code": '//*[@id="imgCode"]',
"code_image_url": '//*[@id="root"]/div/div/div[2]/div/form/div[3]/div/div/span/div/div[2]/img',
"success_ele": '//*[@id="root"]/div/div/div[1]/div/div/span/span'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (455, 315, 578, 355), "30400")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
print(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"https://www.souwl.cn/admin/register/index.shtml?start_time={self.today}+00%3A00%3A00&end_time={self.tomorrow}+00%3A00%3A00&channel=337"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
selector = Selector(text=response.text)
# 获取数据
info = selector.xpath('/html/body/table[1]/tbody/tr/td/text()').extract()
# 获取结果
result = {
"注册人数": info[0],
"实名人数": "null",
"申请人数": "null",
"放款人数": info[1]
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button":
'//*[@id="root"]/div/div/form/div[3]/div/div/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = "http://c.ypuzhen.com/web/clUser/selectListAll"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
form = {
'pageNum': 1,
'pageSize': 2000,
'channelId': 284,
'startTime': f'{self.today}',
'endTime': f'{self.today}',
}
# 访问url
response = session.post(url, headers=headers, data=form)
json_info = response.json()['data']['pageDto']
#计算实名人数
num = json_info['list']
real_name = []
for i in num:
name = i['realName']
if name:
real_name.append(name)
result = {
"注册人数": json_info["total"],
"实名人数": len(real_name),
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button":
'//*[@id="root"]/div/div/form/div[3]/div/div/button',
"check_code": '',
"code_image_url": '',
"success_ele": ''
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# page的url
page_url = "http://xfshchannel.tgjrfw.com/web/clUser/selectListAll"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36",
}
form = {
"pageNum": "1",
"pageSize": "2000",
"channelId": "396",
"startTime": f"{self.today}",
"endTime": f"{self.today}"
}
# 访问url
response = session.post(page_url, headers=headers, data=form)
info = response.json()["data"]["pageDto"]
# 最终结果
result = {
"注册人数": info['total'],
"实名人数": 0,
"申请人数": 'null',
"放款人数": 'null',
"备注": ""
}
sm = info["list"]
for each in sm:
# print(each)
if each["realName"]:
result["实名人数"] += 1
# print(result)
self.write_sql(result)
def get_info(self):
xpath_info = {
"username":
'******',
"password":
'******',
"login_button":
'//*[@id="login-page-full"]/div/div[3]/form/ul/li[4]/button',
"check_code":
'//*[@id="login-page-full"]/div/div[3]/form/ul/li[3]/input',
"code_image_url":
'//*[@id="login-page-full"]/div/div[3]/form/ul/li[3]/span/img',
"success_ele": '//*[@id="header-navbar"]/div/div[1]/ul/li/a/img'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (616, 405, 710, 438),
"20300")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://0v10.cn/admin/customer/promote.html?start_time={self.today.year}%2F{str(100+int(self.today.month))[1:]}%2F{str(100+self.today.day)[1:]}+-+{self.tomorrow.year}%2F{str(100+int(self.tomorrow.month))[1:]}%2F{str(100+self.tomorrow.day)[1:]}"
print(page_url)
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
selector = Selector(text=response.text)
# 获取数据
info = selector.xpath(
'//*[@id="content-wrapper"]/div/div/div[2]/div/div/div/div[2]/form/table//tr/td[4]/text()'
).extract()[0]
print(info)
# 获取结果
result = {
"注册人数": info,
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
'备注': ''
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="login-form"]/div/button',
"check_code": '',
"code_image_url": '',
"success_ele": ''
}
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 将cookie设置给session
session = Session()
# 设置session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
url = f"https://ssda.xyshuj.com/?start_date={self.today}&end_date={self.today}"
# 请求url
response = session.get(url, headers=headers)
# 获取结果
html = Selector(text=response.text)
result = {
"注册人数":
html.xpath(
'//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[3]/text()'
).extract()[0],
"实名人数":
"null",
"申请人数":
html.xpath(
'//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[4]/text()'
).extract()[0],
"放款人数":
html.xpath(
'//*[@id="main-content"]/section/section/div[2]/div/div[2]/table//tr[2]/td[5]/text()'
).extract[0],
"备注":
""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button":
'//*[@id="root"]/div/div/form/div[3]/div/div/span/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
print(cookie)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = "http://c.wedom.cn/web/clUser/selectUserCount"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
arg = {
'channelId': "528",
'endTime': f"{self.today}",
'pageNum': 1,
'pageSize': 20,
'startTime': f"{self.today}",
}
# 访问url
response = session.post(url, headers=headers, json=arg)
info = response.json()['data'][0]
print(info)
result = {
"注册人数": info['countUser'],
"实名人数": "null",
"申请人数": info['countBorrow'],
"放款人数": info['countSuccessBorrow'],
"备注": ''
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="login"]/form/button',
"check_code": '',
"code_image_url": '',
"success_ele": ''
}
# 设置session
session = Session()
# 获取cookie
cookie = cookie_to_dict(self.no_check_get_cookie(xpath_info))
# cookie["pagenum"] = "200"
# 将cookie设置给session
session.cookies.update(cookie)
# print(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
json_url = "https://toupin.cn/qianbaoadm/channel/list"
# 请求url
#post
arg = {
'addtime_endtime': f'{self.today}',
'addtime_starttime': f'{self.tomorrow}',
'currentpage': "1"
}
response = session.post(json_url,
headers=headers,
json=json.dumps(arg))
# print(response.text)
# print(response.status_code)
info = response.json()['data']
# 获取结果
result = {
"注册人数": info['listnum'],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="logina"]',
"check_code": '',
"code_image_url": '',
"success_ele": ''
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = "http://haoshun.youdaikeji.com/mgmt/thirdChannel/channelStatistics?channelId=94"
#参数
arg = {
'startDateTime': f'{self.today}',
# 'channelId':,
'page': 1,
'rows': 10,
}
# 请求url
response = session.post(page_url, headers=headers, data=arg)
# 构造Selector
data = response.json()['rows'][0]
print(data)
# 获取结果
result = {
"注册人数": data['countUser'],
"实名人数": 'null',
"申请人数": data['countApply'],
"放款人数": "null"
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div/form/input[4]',
"check_code": '/html/body/div/form/input[3]',
"code_image_url": '/html/body/div/form/img',
"success_ele": '/html/body/div[1]/ul/li[2]/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (502, 390, 638, 439),
"10400")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = "http://www.lovezcm.com/ymd/admin/channel-view.jsp?id=0.7025606997962637"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
html = Selector(text=response.text)
# 获取结果
result = {
"注册人数":
html.xpath('/html/body/div[1]/div[2]/table//tr[2]/td[3]/text()').
extract()[0],
"实名人数":
"null",
"申请人数":
"null",
"放款人数":
"null",
"备注":
""
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="ju"]/form/button',
"check_code": '//*[@id="ju"]/form/div[3]/input',
"code_image_url": '//*[@id="c2c98f0de5a04167a9e427d883690ff6"]',
"success_ele": '/html/body/div/header/a/span[2]'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (467, 569, 619, 621),
"30500")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://openback.hongzigame.com/channel/getChannelStatistics?channelName=&channelCode={self.channel}&startDate={self.today}&endDate=&numPerPage=10&pageNum=1&xzOldPageNum=1"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
selector = Selector(text=response.text)
# 获取数据
info = selector.xpath(
'//*[@id="bigDataList"]/tbody/tr/td/text()').extract()
print(info)
# 获取结果
result = {
"注册人数": info[4],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="loginBtn"]',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = 'http://channel.mgshujia.com/api?cmd=chluserTotal'
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
#postn内容
arg = {
'cnds':
f'%7B%22date_FROM_cnd%22%3A%22{self.today}%22%2C%22date_TO_cnd%22%3A%22{self.today}%22%2C%22channel_path_cnd%22%3A%22%22%2C%22is_fuzzy_cnd%22%3A%22%22%7D'
}
# 访问url
response = session.post(url, headers=headers, data=arg)
# print(response.status_code)
# print(response.json())
json_info = response.json()["rows"]
result = {
"注册人数": json_info["register_num"],
"实名人数": json_info['complete_info_num'],
"申请人数": 'null',
"放款人数": json_info['credit_pass_num']
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="form"]/div[4]/div/a[1]',
"check_code": '//*[@id="form"]/div[3]/div/input',
"code_image_url": '//*[@id="verify_img"]',
"success_ele": '//*[@id="推广产品"]/span'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (984, 454, 1148, 503), "10400")
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# page的url
page_url = f"http://s.51jinkong.cn/admin/channelmenu/index?shop_id=86&the_year_month={self.today.year}-{self.today.month}"
# 设置头部
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(page_url, headers=headers)
# 获取html
html = response.text
# 构造选择器
selector = Selector(text=html)
# 获取数据
info = selector.xpath('/html/body/div[4]/div/article/table[2]/tbody/tr[1]')
registers = info.xpath('./td[2]/text()').extract()[0].strip()
apply = info.xpath('./td[3]/font/text()').extract()[0]
success = info.xpath('.//td[7]/font/text()').extract()[0]
# 最终结果
result = {
"注册人数": registers,
"实名人数": "null",
"申请人数": apply,
"放款人数": success
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="loginForm"]/div[5]/div/input',
"check_code": '//*[@id="captcha"]',
"code_image_url": '//*[@id="loginForm"]/div[4]/div[2]/img',
"success_ele": '//*[@id="menu"]/li/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (300, 376, 600, 405),
"30400")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = 'http://back.yzb668.com/diversion/statistics'
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
selector = Selector(text=response.text)
# 获取数据
# print(selector)
info = selector.xpath(
'//*[@id="bigDataList"]/tbody/tr/td/text()').extract()
# 获取结果
result = {
"注册人数": info[2],
"实名人数": info[3],
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div/div/div/form/div[4]',
"check_code": '//*[@id="captcha"]',
"code_image_url": '//*[@id="validateCodeImg"]',
"success_ele": '/html/body/div/div[1]/div[2]/ul'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (507, 487, 638, 530),
"10400")
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# page的url
page_url = "http://partner.ytrong.com/partner/info?id=310&page=1&limit=30"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
arg = {'id': 310, 'page': 1, 'limit': 30}
# 访问url
response = session.post(page_url, headers=headers, data=arg)
# 获取html
info = response.json()['rows'][0]
# print(info)
# 最终结果
result = {
"注册人数": info['userCount'],
"实名人数": "null",
"申请人数": 'null',
"放款人数": 'null',
"备注": ""
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="app"]/div/form/div[3]/div/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
json_url = f"http://crm.channel.51zhihe.com/backend/statistics?page=1&data=%7B%22search%22:%7B%22start%22:%22{self.today}%22,%22end%22:%22{self.today}%22,%22name%22:%22%22%7D%7D"
# 设置头部
headers = {
"Host":
f"{cookie[0]['domain']}",
"Referer":
f"http://{cookie[0]['domain']}/backend/index",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(json_url, headers=headers)
# 获取数据
json_info = response.json()["data"]["lists"]["data"][0]
# 最终结果
result = {
"注册人数": json_info["registers"],
"实名人数": "null",
"申请人数": json_info["applications"],
"放款人数": "null"
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="rrapp"]/div[2]/div[4]/div[2]/button',
"check_code": '//*[@id="rrapp"]/div[2]/div[3]/input',
"code_image_url": '//*[@id="rrapp"]/div[2]/div[3]/img',
"success_ele": '//*[@id="rrapp"]/header/nav/div[1]'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (525, 305, 633, 340),
"30500")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://106.14.158.253:8090//getRegisterCount?_search=false&nd={str(int(time.time()*1000))}&size=10¤t=1&sidx=&order=asc&startTime={self.today}&endTime={self.today}&channelCode=ka10&_={str(int(time.time()*1000))}"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
data = response.json()['records'][0]
print(data)
# 获取结果
result = {
"注册人数": data['registerCount'],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div/div/div/form/div[4]',
"check_code": '//*[@id="captcha"]',
"code_image_url": '//*[@id="validateCodeImg"]',
"success_ele": '/html/body/div/div[1]/div[2]/ul/li[1]/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (504, 492, 635, 535),
"10400")
# 将cookie设置给session
session.cookies.update(cookie_to_dict(cookie))
# 设置头部信息
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 页面url
page_url = f"http://partner.ytrong.com/partner/info?id=354&page=1&limit=30"
# 请求url
response = session.get(page_url, headers=headers)
# 构造Selector
data = response.json()['rows'][0]
print(data)
# 获取结果
result = {
"注册人数": data['userCount'],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null"
}
self.write_sql(result)
print(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '/html/body/div/div[2]/form/div[3]/div[2]/button',
"check_code": "",
"code_image_url": "",
"success_ele": ""
}
# 设置session
session = Session()
# 获取cookie
cookie = self.no_check_get_cookie(xpath_info)
print(cookie)
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# json的url
url = f"http://report.xdy7.com/user/leads/daily?source={self.channel}&datebegin={self.today}&dateend={self.today}"
# 设置头部
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(url, headers=headers)
info = Selector(text=response.text)
result = {
"注册人数": info.xpath('//table//tr/td[2]/text()').extract()[0],
"实名人数": "null",
"申请人数": "null",
"放款人数": "null",
"备注": ''
}
self.write_sql(result)
def get_info(self):
xpath_info = {
"username": '******',
"password": '******',
"login_button": '//*[@id="loginForm"]/div[5]/button',
"check_code": '//*[@id="verify"]',
"code_image_url": '//*[@id="verifyImg"]',
"success_ele": '//*[@id="navbar-header"]/ul[1]/li[3]/a'
}
# 设置session
session = Session()
# 获取cookie
cookie = self.check_get_cookie(xpath_info, (563, 486, 656, 522), "30400")
# 给session设置cookie
session.cookies.update(cookie_to_dict(cookie))
# page的url
page_url = "https://quan.jsd0086.com/agent/report/index"
# 设置头部
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36",
}
# 访问url
response = session.get(page_url, headers=headers)
# 获取html
info = Selector(text=response.text)
print(response.text)
# 最终结果
result = {
"注册人数": info.xpath('//table//tr/td[2]/text()').extract()[0],
"实名人数": info.xpath('//table//tr/td[5]/text()').extract()[0],
"申请人数": 'null',
"放款人数": 'null',
"备注": ""
}
self.write_sql(result)
