def TestProxt(proxy, time):
try:
pr = urllib3.ProxyManager('http://' + proxy)
pr.request('GET', 'https://www.facebook.com', timeout=time)
except:
try:
prs = SOCKSProxyManager('socks5://' + proxy)
prs.request('GET', 'https://www.facebook.com', timeout=time)
except:
return 0
else:
return 1
else:
return 1
def run(self):
while not self.queue.empty():
work = self.queue.get()
http = SOCKSProxyManager('socks5://127.0.0.1:9150')
# http = urllib3.PoolManager(cert_reqs ='CERT_REQUIRED',ca_certs=certifi.where())
try:
httpResponse = http.request('GET', baseUrl + work)
except:
print("Error")
if len(httpResponse.data) != 4118:
print(
f'---------------------------------------------------------------------------------------------'
)
print(
f'-******************************************************************************************- '
)
print(
f'-*************************** {work} Is what you want **************************************- '
)
print(
f'-******************************************************************************************- '
)
print(
f'---------------------------------------------------------------------------------------------'
)
print("--- %s seconds ---" % (time.time() - start_time))
self.queue.task_done()
break
else:
self.queue.task_done()
def checkLink(link):
try:
global siteTitle
proxy = SOCKSProxyManager('socks5h://localhost:9050/')
websiteObj = proxy.request('GET', link, timeout=3.0)
response_code = websiteObj.status
sitedata = websiteObj.data
soup = BeautifulSoup(sitedata, "html.parser")
siteTitle = soup.title.string
except:
response_code = 500
pass
if response_code >= 200 and response_code < 404:
status = 'OK'
print("Title:", bcolors.WARNING + siteTitle + bcolors.ENDC)
print("Link: ", bcolors.OKBLUE + link + bcolors.ENDC, "\nStatus:",
bcolors.OKGREEN + status + bcolors.ENDC)
c.execute(
'''INSERT INTO onionsites (url, title, status, status_code) VALUES (?, ?, ?, ?);''',
(str(link), str(siteTitle), str(status),
int(response_code))) # Storing all the results
conn.commit()
elif response_code == 500:
status = 'Failed'
print("Status code:", response_code)
print("Link: ", bcolors.FAIL + str(link) + bcolors.ENDC, "\nStatus:",
bcolors.FAIL + status + bcolors.ENDC)
def checkIPs():
"""
This funciton checks first the Public IP and then the TOR IP and prints them out.
"""
proxy = SOCKSProxyManager('socks5h://localhost:9050/')
http = urllib3.PoolManager()
myip = http.request('GET', 'http://httpbin.org/ip')
try:
global torIP
torIP = proxy.request('GET', 'http://httpbin.org/ip')
except NameError:
print(
bcolors.FAIL +
"proxy isn't running or address:port is wrong.\nScript will exit."
+ bcolors.ENDC)
exit(1)
jsonIP = json.loads(myip.data.decode('utf-8'))
jsonTorIP = json.loads(torIP.data.decode('utf-8'))
print("Your real IP Address is: " + bcolors.OKBLUE + jsonIP["origin"] +
bcolors.ENDC)
print("Your Tor IP Address is: " + bcolors.FAIL + jsonTorIP["origin"] +
bcolors.ENDC)
print(bcolors.FAIL + "=" * 50 + bcolors.ENDC)
def resolveJson(path):
headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36'
}
proxy = SOCKSProxyManager('socks5://localhost:1080/', headers=headers)
response = proxy.request(
'get',
"https://storage.googleapis.com/fantasylauncer/wallpaper/wallpaperOnline.json"
)
with open("wallpaperOnline.json", 'wb+') as f:
f.write(response.data)
f.close()
def saver(url):
mgr = SOCKSProxyManager(
'socks5://%s:%s/' %
(config.get('Proxy', 'host'), config.get('Proxy', 'port')), **{
'username': config.get('Proxy', 'username'),
'password': config.get('Proxy', 'password')
})
tordata = mgr.request('GET', url, headers={
'connection': 'keep-alive'
}).data
out_dir = config.get('Torrent', 'save_dir')
with open(os.path.join(out_dir, os.path.basename(url)), 'wb') as torfile:
torfile.write(tordata)
return tordata
def get_data(uri, instrument_id, num):
unixtime = int(time.time())
sign_str = '%d%s%s%s' % (unixtime, method, uri, secretkey)
# 签名
sha256 = hashlib.sha256(
sign_str.encode('utf-8')).hexdigest().encode('utf-8')
signature_str = base64.b64encode(sha256).decode('utf-8')
#print(sign_str.encode('utf-8'))
#print(sha256)
#print(signature_str)
headers = {
'OK-ACCESS-KEY': apikey,
'OK-ACCESS-SIGN': signature_str,
'OK-ACCESS-TIMESTAMP': '%d' % unixtime,
'OK-ACCESS-PASSPHRASE': Passphrase,
'Accept': 'application/json',
'Content-Type': 'application/json; charset=UTF-8',
}
# 发起调用
url = host + uri
print(url)
start_time = datetime.now()
# 不使用代理
#pool = urllib3.PoolManager(num_pools=2, timeout=180, retries=False)
#r = pool.urlopen(method, url, headers=headers)
# 使用sock5代理
proxy = SOCKSProxyManager('socks5h://localhost:1080/')
r = proxy.request(method, url, headers=headers)
print('[Time taken: {!s}]'.format(datetime.now() - start_time))
#print(r.status)
#print(r.data)
if r.status != 200:
print("fail: ", r.status, r.data)
return []
# 获取数据，截断为num， 注意顺序：最新数据在前
data = json.loads(r.data.decode('utf-8'))
data = data[:num]
return data
class TorProxy:
def __init__(self):
self.proxy = SOCKSProxyManager('socks5://127.0.0.1:9050')
def new_ip(self):
with Controller.from_port(port=9051) as controller:
controller.authenticate(password="******")
controller.signal(Signal.NEWNYM)
controller.close()
def request(self, url):
self.new_ip()
headers = {'User-Agent': UserAgent().random}
rq = self.proxy.request('GET', url, headers=headers)
return rq
def check_proxy(proxy, proxy_type):
if proxy_type == 'socks':
pool = SOCKSProxyManager('socks5://%s' % proxy)
else:
pool = urllib3.ProxyManager('http://%s' % proxy)
retries = urllib3.Retry(total=None,
connect=False,
read=False,
redirect=10,
raise_on_redirect=False)
timeout = urllib3.Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT)
op = {
'status': None,
'connect_time': None,
'read_time': None,
'error': None,
}
try:
start_time = time.time()
res = pool.request('GET',
'https://en.wikipedia.org/robots.txt',
retries=retries,
timeout=timeout,
preload_content=False)
connected_time = time.time()
op['connect_time'] = round(connected_time - start_time, 2)
data = res.read()
op['read_time'] = round(time.time() - connected_time, 2)
except Exception as ex:
error = type(ex).__name__
op['error'] = error
if error in ('NewConnectionError', 'ConnectTimeoutError'):
op['status'] = 'connect_fail'
elif error in (
'ProtocolError',
'ReadTimeoutError',
):
op['status'] = 'read_fail'
else:
raise Exception('Unexpected error: %s' % error)
else:
if b'# Wikipedia work bots:' in data:
op['status'] = 'ok'
else:
op['status'] = 'data_fail'
return op
def pageRequest(url):
global roundRobin
proxy = SOCKSProxyManager('socks5://localhost:'+str(torPort),
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
headers={'user-agent': randomUserAgent(), 'Cookie': ''})
http = urllib3.PoolManager( 1,
cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where(),
headers={'user-agent': randomUserAgent(), 'Cookie': ''})
if roundRobin % 2:
response = http.request('GET', url)
else:
if torSupport:
response = proxy.request('GET', url)
else:
response = http.request('GET', url)
roundRobin += 1
if not roundRobin % 60:
newTorIdentity()
return response.data
def query(url, sport):
"""Uses urllib3 to fetch a site using the proxy on the @sport. """
proxy = SOCKSProxyManager('socks5://localhost:{}/'.format(sport))
r = proxy.request('GET', url)
return r.data
def prepare_options(self):
if is_blank(self._url):
raise Exception("GoogleArtsCrawlerOption , url is blank!")
uprs = parse_url(url=self._url)
if not uprs.host == 'artsandculture.google.com':
raise Exception("GoogleArtsCrawlerOption, url netloc is not `artsandculture.google.com`")
self._url = "https://{0}{1}".format(uprs.host, uprs.path)
# download webdriver
if self._webdriver_execute_path is None and self._need_download_webdrive:
default_webdrive_path = "webdriver"
if os.path.isdir(default_webdrive_path):
default_webdrive_files = os.listdir(default_webdrive_path)
if len(default_webdrive_files) > 0:
default_webdrive_execute_file = os.path.join(default_webdrive_path, default_webdrive_files[0])
if os.path.isfile(default_webdrive_execute_file):
print("==> webdriver has exist at {0}".format(default_webdrive_execute_file))
self._webdriver_execute_path = default_webdrive_execute_file
if self._webdriver_execute_path is None:
if WINDOWS:
os_name = "Windows"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_win32.zip"
elif DARWIN:
os_name = "Mac OS"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_mac64.zip"
elif LINUX:
os_name = "Linux"
webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_linux64.zip"
else:
raise Exception("GoogleArtsCrawlerOptions, unknown platform !")
print("==> current operation system : {0}".format(os_name))
print("==> prepare download webdriver : {0}".format(webdriver_download_url))
default_download_tmp = "tmp"
webdriver_zip_filename = webdriver_download_url.split("/")[-1]
webdriver_local_zip_filepath = os.path.join(default_download_tmp, webdriver_zip_filename)
# not exist
if not os.path.isfile(webdriver_local_zip_filepath):
http = SOCKSProxyManager('socks5://localhost:1086/')
# http = PoolManager()
response = http.request('GET', webdriver_download_url, preload_content=False)
if not os.path.isdir(default_download_tmp):
os.mkdir(default_download_tmp)
with open(webdriver_local_zip_filepath, mode="wb") as fd:
while True:
data = response.read(1024)
if not data:
break
fd.write(data)
response.release_conn()
print("==> webdriver zip file download finished , location at : {0}".format(
os.path.abspath(webdriver_local_zip_filepath)))
else:
print("==> webdriver zip file has existed at {0}".format(webdriver_local_zip_filepath))
with ZipFile(webdriver_local_zip_filepath, 'r') as zipfile:
zipfile.extractall(path=default_webdrive_path)
if self._need_clear_cache:
shutil.rmtree(default_download_tmp)
self._webdriver_execute_path = os.path.join(default_webdrive_path, os.listdir(default_webdrive_path)[0])
if is_blank(self._webdriver_execute_path):
raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is blank!")
if not os.path.isfile(self._webdriver_execute_path):
raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is not exist, this is file!")
if LINUX or DARWIN:
os.chmod(self._webdriver_execute_path, 0o777)
# self._chrome_options.binary_location = os.path.abspath(self._webdriver_execute_path)
mobile_emulation = {
"deviceMetrics": {"width": self._size, "height": self._size, "pixelRatio": 1.0},
"userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 "
"(KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19"}
self._chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
self._chrome_options.add_argument('--no-sandbox')
self._chrome_options.add_argument('--disable-dev-shm-usage')
self._chrome_options.add_argument('--disable-gpu')
self._chrome_options.add_argument("--disable-dev-shm-usage")
self._chrome_options.add_argument("start-maximized")
self._chrome_options.add_argument("disable-infobars")
self._chrome_options.add_argument("--disable-extensions")
if not self._is_debug:
self._chrome_options.add_argument("--headless")
self._output_path = DEFAULT_GCO_OUTPUT_PATH if self._output_path is None else self._output_path
self._size = DEFAULT_GCO_SIZE if self._size is None or self._size < 1 else self._size
self._init_delay_time = DEFAULT_GCO_INIT_DELAY if self._init_delay_time is None or self._init_delay_time < 1 else self._init_delay_time
if not os.path.isdir(self._output_path):
os.makedirs(self._output_path)
if not os.path.isdir(self._partial_tmp_path):
os.makedirs(self._partial_tmp_path)
if self._is_debug:
print("GoogleArtsCrawlerOptions:")
print("==> url:{0}".format(self._url))
print("==> webdriver_execute_path:{0}".format(os.path.abspath(self._webdriver_execute_path)))
print("==> output :{0}".format(os.path.abspath(self._output_path)))
return self
def test_socks_proxy(proxy,time):
prs = SOCKSProxyManager('socks5://'+proxy)
prs.request('GET', 'https://www.youtube.com', timeout=time)
class Request(Singleton):
__request_timeout = 180
__allow_redirects = False
__default_user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0'
def __parse_custom_header(self, custom_header):
parsed_custom_header = custom_header.split(':')
custom_header_key = parsed_custom_header[0].strip()
custom_header_value = parsed_custom_header[1].strip()
self.__headers[custom_header_key] = custom_header_value
def __init__(self,
url,
user_agent,
cookies_string=False,
custom_header=False,
insecure_ssl='false',
proxy=False):
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
self.__url = url
self.__headers = dict()
self.__headers[
'User-Agent'] = self.__default_user_agent if user_agent == 'default' else user_agent
if cookies_string:
self.__headers['Cookie'] = cookies_string
if custom_header:
self.__parse_custom_header(custom_header)
self.__verify = 'CERT_REQUIRED' if insecure_ssl == 'false' else 'CERT_NONE'
if proxy:
proxy_type = proxy.split('://')[0]
if proxy_type == 'http' or proxy_type == 'https':
self.__request_obj = urllib3.ProxyManager(
proxy,
ssl_version=ssl.PROTOCOL_TLSv1,
timeout=self.__request_timeout,
cert_reqs=self.__verify)
else:
self.__request_obj = SOCKSProxyManager(
proxy,
ssl_version=ssl.PROTOCOL_TLSv1,
timeout=self.__request_timeout,
cert_reqs=self.__verify)
else:
self.__request_obj = urllib3.PoolManager(
ssl_version=ssl.PROTOCOL_TLSv1,
timeout=self.__request_timeout,
cert_reqs=self.__verify)
# print (vars(self))
def send_request(self, request):
try:
response_object = self.__request_obj.request(
'POST',
self.__url,
fields={'data': request},
redirect=self.__allow_redirects,
headers=self.__headers,
retries=False)
except KeyboardInterrupt:
raise Exception('Keyboard interrupt issued')
return response_object.status, response_object.headers, response_object.data
def test_socks_proxy(proxy):
pr = SOCKSProxyManager('socks5://'+proxy)
pr.request('GET', 'https://www.youtube.com')