def getHTMLText(self, url, header, goods_root, index, sort):
select_random = '''
select ip,port,proxy_type from proxys order by rand() limit 1
'''
cursor.execute(select_random)
conn.commit()
result = cursor.fetchone()
if result is None:
getIP.crawl_ips()
global tryTime
if tryTime == 1:
return False
else:
tryTime = 1
self.getHTMLText(url, header, goods_root, index, sort)
else:
ip = result[0]
port = result[1]
type = result[2]
judge_re = self.judge_ip(ip, port, type, url, header, goods_root, index, sort)
if judge_re:
return True
else:
self.getHTMLText(url, header, goods_root, index, sort)
return False
def getHTMLText(self, id):
try:
select_random = '''
select ip,port,proxy_type from proxys order by rand() limit 1
'''
cursor.execute(select_random)
conn.commit()
result = cursor.fetchone()
if result is None:
getIP.crawl_ips()
global tryTime
if tryTime == 1:
return False
else:
tryTime = 1
self.getHTMLText(id)
else:
ip = result[0]
port = result[1]
type = result[2]
judge_re = self.judge_ip(ip, port, type, id)
if judge_re:
return True
else:
self.getHTMLText(id)
return False
except Exception as e:
print(e)