def get_xiladaili_run():
xiladaili_url = "http://www.xiladaili.com/"
xiladaili_html_1 = mreq.ManGeReq.geturl(xiladaili_url)
xiladaili_ipTbody1 = '/html/body/div/div[3]/div[@id="scroll"]'
xiladaili_ipTbody_datas1 = rpub.PubXpath.get_html_tonumber(
xiladaili_html_1, xiladaili_ipTbody1, 0, 3)
xiladaili_ip_table = rpub.get_table(xiladaili_ipTbody_datas1)
print(len(xiladaili_ip_table))
for xiladaili_ip_lists in xiladaili_ip_table:
for xiladaili_ip_infos in rpub.get_onlytr_data(xiladaili_ip_lists)[2:]:
xiladaili_iptd = rpub.get_onlytd_data(xiladaili_ip_infos)
httptype = xiladaili_iptd[2]
ipnumber = xiladaili_iptd[0]
iphttpval = httptype.split(',')
if len(iphttpval) == 2:
ipproxy1 = iphttpval[0].lower() + "://" + ipnumber
print(ipproxy1)
ipproxy2 = iphttpval[1].lower() + "://" + ipnumber
print(ipproxy1)
toredis.add(ipproxy1)
toredis.add(ipproxy2)
else:
ipproxy1 = iphttpval[0].lower() + "://" + ipnumber
print(ipproxy1)
toredis.add(ipproxy1)
def get_jiangxianli_run():
jiangxianli_pg = 1
while jiangxianli_pg < 4:
jiangxianli_url = "http://ip.jiangxianli.com/?page=" + str(
jiangxianli_pg)
jiangxianli_html_1 = mreq.ManGeReq.geturl(jiangxianli_url)
jiangxianli_ipTbody1 = '/html/body/div/div[2]/div[1]/div[1]/table/tbody'
jiangxianli_ipTbody_datas1 = rpub.PubXpath.get_html(
jiangxianli_html_1, jiangxianli_ipTbody1)
for jiangxianli_ip_infos in rpub.get_onlytr_data(
jiangxianli_ipTbody_datas1):
jiangxianli_iptd = rpub.get_onlytd_data(jiangxianli_ip_infos)
ipproxy = jiangxianli_iptd[3].lower(
) + "://" + jiangxianli_iptd[0] + ":" + jiangxianli_iptd[1]
print(ipproxy)
toredis.add(ipproxy)
jiangxianli_pg += 1
def get_ip3366_run():
ip3366_pg = 1
while ip3366_pg < 7:
ip3366_url = "http://www.ip3366.net/free/?stype=1&page=" + str(
ip3366_pg)
ip3366_html = mreq.ManGeReq.geturl(ip3366_url)
ip3366_ipTbody = '//div[@id="list"]/table/tbody'
ip3366_ipTbody_datas = rpub.PubXpath.get_html(ip3366_html,
ip3366_ipTbody)
for ip3366_ip_infos in rpub.get_onlytr_data(ip3366_ipTbody_datas):
ip3366_iptd = rpub.get_onlytd_data(ip3366_ip_infos)
ipproxy = ip3366_iptd[3].lower(
) + "://" + ip3366_iptd[0] + ":" + ip3366_iptd[1]
print(ipproxy)
toredis.add(ipproxy)
ip3366_pg += 1
time.sleep(1)
def get_qydaili_run():
qydaili_pg = 1
while qydaili_pg < 11:
qydaili_url = "http://www.qydaili.com/free/?action=china&page=" + str(
qydaili_pg)
qydaili_html = mreq.ManGeReq.geturl(qydaili_url)
qydaili_ipTbody = '//div[@class="container"]/table/tbody'
qydaili_ipTbody_datas = rpub.PubXpath.get_html(qydaili_html,
qydaili_ipTbody)
for qydaili_ip_infos in rpub.get_onlytr_data(qydaili_ipTbody_datas):
qydaili_iptd = rpub.get_propertytd_data(qydaili_ip_infos)
ipproxy = qydaili_iptd[3].lower(
) + "://" + qydaili_iptd[0] + ":" + qydaili_iptd[1]
print(ipproxy)
toredis.add(ipproxy)
time.sleep(1)
qydaili_pg += 1
def get_w66ip_run():
w66ip_pg = 1
while w66ip_pg < 1234:
w66ip_url = "http://www.66ip.cn/" + str(w66ip_pg) + ".html"
w66ip_html = mreq.ManGeReq.geturl(w66ip_url)
w66ip_ipTbody = '//*[@id="main"]/div/div[1]/table'
w66ip_ipTbody_datas = rpub.PubXpath.get_html(w66ip_html, w66ip_ipTbody)
for w66ip_ip_infos in rpub.get_onlytr_data(w66ip_ipTbody_datas)[2:]:
w66ip_iptd = rpub.get_onlytd_data(w66ip_ip_infos)
ipproxy_1 = "http://" + w66ip_iptd[0] + ":" + w66ip_iptd[1]
ipproxy_2 = "https://" + w66ip_iptd[0] + ":" + w66ip_iptd[1]
print(ipproxy_1)
print(ipproxy_2)
toredis.add(ipproxy_1)
toredis.add(ipproxy_2)
time.sleep(1)
w66ip_pg += 1
def get_iphai_run():
iphai_url_list = [
"http://www.iphai.com/free/ng", "http://www.iphai.com/free/wg"
]
for iphai_url in iphai_url_list:
iphai_html = mreq.ManGeReq.geturl(iphai_url)
iphai_ipTbody = '/html/body/div[2]/div[2]/table'
iphai_ipTbody_datas = rpub.PubXpath.get_html(iphai_html, iphai_ipTbody)
for iphai_ip_infos in rpub.get_onlytr_data(iphai_ipTbody_datas)[1:]:
iphai_iptd = rpub.get_onlytd_data(iphai_ip_infos)
ipnumber = (iphai_iptd[0].replace("\n", "")).split()[0]
port = (iphai_iptd[1].replace("\n", "")).split()[0]
ipproxy_1 = "http://" + ipnumber + ":" + port
ipproxy_2 = "https://" + ipnumber + ":" + port
print(ipproxy_1)
print(ipproxy_2)
toredis.add(ipproxy_1)
toredis.add(ipproxy_2)
def get_w89ip_run():
w89ip_pg = 1
while w89ip_pg < 7:
w89ip_url = "http://www.89ip.cn/index_" + str(w89ip_pg) + ".html"
w89ip_html = mreq.ManGeReq.geturl(w89ip_url)
w89ip_ipTbody = '//table[@class="layui-table"]/tbody'
w89ip_ipTbody_datas = rpub.PubXpath.get_html(w89ip_html, w89ip_ipTbody)
for w89ip_ip_infos in rpub.get_onlytr_data(w89ip_ipTbody_datas):
w89ip_iptd = rpub.get_onlytd_data(w89ip_ip_infos)
ipnumber = re.sub('(\n\t\t\t)|(\t\t)', '', w89ip_iptd[0])
port = re.sub('(\n\t\t\t)|(\t\t)', '', w89ip_iptd[1])
ipproxy_1 = "http://" + ipnumber + ":" + port
ipproxy_2 = "https://" + ipnumber + ":" + port
print(ipproxy_1)
print(ipproxy_2)
toredis.add(ipproxy_1)
toredis.add(ipproxy_2)
time.sleep(1)
w89ip_pg += 1
def get_crossincode_run():
crossincode_url = "https://lab.crossincode.com/proxy/"
crossincode_html_1 = mreq.ManGeReq.geturl(crossincode_url)
crossincode_ipTbody = '/html/body/div/div/div[2]/div[2]/table'
crossincode_ipTbody_datas = rpub.PubXpath.get_html(crossincode_html_1,
crossincode_ipTbody)
crossincode_iplist = rpub.get_onlytr_data(crossincode_ipTbody_datas)
for crossincode_ipinfo in crossincode_iplist[1:]:
crossincode_iptd = rpub.get_onlytd_data(crossincode_ipinfo)
httptype = crossincode_iptd[3]
ipnumber = crossincode_iptd[0]
ipport = crossincode_iptd[1]
iphttpval = httptype.split(',')
if len(iphttpval) == 2:
ipproxy1 = iphttpval[0].lower() + "://" + ipnumber + ":" + ipport
print(ipproxy1)
ipproxy2 = iphttpval[1].lower() + "://" + ipnumber + ":" + ipport
print(ipproxy1)
toredis.add(ipproxy1)
toredis.add(ipproxy2)
else:
ipproxy1 = iphttpval[0].lower() + "://" + ipnumber + ":" + ipport
print(ipproxy1)
toredis.add(ipproxy1)