def get_address(proxy):
"""fetch american address from https://fakena.me/random-real-address/
Args:
proxy (str): proxy to visit the target site, ip:port
Returns:
format_addr (str): american address in the form of "address_line # city # state # zip"
"""
ignore_warnings()
url = r'https://fakena.me/random-real-address/'
referer = r'https://fakena.me'
header = {'user-agent': generate_user_agent(), 'referer': referer}
curr_proxy = {'http': 'http://%s' % proxy}
text = requests.get(url, headers=header, proxies=curr_proxy).text
pattern = re.compile('<strong>(.+)<br>(.+)</strong>')
result = re.findall(pattern, text)
if result: # sometimes the result is empty
print result[0][0], result[0][1]
address_line = result[0][0]
city, state_zip = result[0][1].split(',')
state, zip = state_zip.split()
format_addr = address_line + '#' + city + '#' + state + '#' + zip
return format_addr
else:
return ''
def get_address(proxy):
"""fetch american address from https://fakena.me/random-real-address/
Args:
proxy (str): proxy to visit the target site, ip:port
Returns:
format_addr (str): american address in the form of "address_line # city # state # zip"
"""
ignore_warnings()
url = r'https://fakena.me/random-real-address/'
referer = r'https://fakena.me'
header = {'user-agent' : generate_user_agent() , 'referer':referer }
curr_proxy ={
'http': 'http://%s'%proxy
}
text = requests.get(url, headers = header, proxies = curr_proxy).text
pattern = re.compile('<strong>(.+)<br>(.+)</strong>')
result = re.findall(pattern, text)
if result: # sometimes the result is empty
print result[0][0], result[0][1]
address_line = result[0][0]
city, state_zip = result[0][1].split(',')
state, zip = state_zip.split()
format_addr = address_line+'#'+city+'#'+state+'#'+zip
return format_addr
else:
return ''
def is_valid(target_url, ip, referer='https://www.google.com'):
"""judge if a proxy ip is valid for target_url
Args:
target_url (str): url that need to visite with a proxy
ip (str): the set in redis to get
referer (str, optional): referer part of headers of the request
Returns:
boolean
"""
ignore_warnings()
proxy = {'http': 'http://%s' % ip}
headers = {'user-agent': generate_user_agent(), 'referer': referer}
try:
r = requests.get(target_url, headers=headers, proxies=proxy, timeout=6)
return True
except Exception:
return False
def is_valid(target_url, ip, referer):
"""judge if a proxy ip is valid for target_url
Args:
target_url (str): url that need to visite with a proxy
ip (str): the set in redis to get
referer (str, optional): referer part of headers of the request
Returns:
boolean
"""
ignore_warnings()
proxy = {
'http': 'http://%s' %ip
}
headers = {'user-agent': generate_user_agent(), 'referer': referer}
try:
r = requests.get(target_url, headers = headers, proxies = proxy, timeout = 6)
return True
except Exception:
return False
sys.exit(0)
def is_valid(target_url, ip, referer='https://www.google.com'):
"""judge if a proxy ip is valid for target_url
Args:
target_url (str): url that need to visite with a proxy
ip (str): the set in redis to get
referer (str, optional): referer part of headers of the request
Returns:
boolean
"""
ignore_warnings()
proxy = {'http': 'http://%s' % ip}
headers = {'user-agent': generate_user_agent(), 'referer': referer}
try:
r = requests.get(target_url, headers=headers, proxies=proxy, timeout=6)
return True
except Exception:
return False
if __name__ == '__main__':
# disable the warnings from https website
ignore_warnings()
while True:
get_proxies(CHINA, 'china_ips', 1, 230)
time.sleep(600)
Args:
target_url (str): url that need to visite with a proxy
ip (str): the set in redis to get
referer (str, optional): referer part of headers of the request
Returns:
boolean
"""
ignore_warnings()
proxy = {
'http': 'http://%s' %ip
}
headers = {'user-agent': generate_user_agent(), 'referer': referer}
try:
r = requests.get(target_url, headers = headers, proxies = proxy, timeout = 6)
return True
except Exception:
return False
if __name__ == '__main__':
# disable the warnings from https website
ignore_warnings()
while True:
get_proxies(CHINA, 'china_ips', 1,230)
time.sleep(600)