def TestProxt(proxy, time): try: pr = urllib3.ProxyManager('http://' + proxy) pr.request('GET', 'https://www.facebook.com', timeout=time) except: try: prs = SOCKSProxyManager('socks5://' + proxy) prs.request('GET', 'https://www.facebook.com', timeout=time) except: return 0 else: return 1 else: return 1
def run(self): while not self.queue.empty(): work = self.queue.get() http = SOCKSProxyManager('socks5://127.0.0.1:9150') # http = urllib3.PoolManager(cert_reqs ='CERT_REQUIRED',ca_certs=certifi.where()) try: httpResponse = http.request('GET', baseUrl + work) except: print("Error") if len(httpResponse.data) != 4118: print( f'---------------------------------------------------------------------------------------------' ) print( f'-******************************************************************************************- ' ) print( f'-*************************** {work} Is what you want **************************************- ' ) print( f'-******************************************************************************************- ' ) print( f'---------------------------------------------------------------------------------------------' ) print("--- %s seconds ---" % (time.time() - start_time)) self.queue.task_done() break else: self.queue.task_done()
def checkLink(link): try: global siteTitle proxy = SOCKSProxyManager('socks5h://localhost:9050/') websiteObj = proxy.request('GET', link, timeout=3.0) response_code = websiteObj.status sitedata = websiteObj.data soup = BeautifulSoup(sitedata, "html.parser") siteTitle = soup.title.string except: response_code = 500 pass if response_code >= 200 and response_code < 404: status = 'OK' print("Title:", bcolors.WARNING + siteTitle + bcolors.ENDC) print("Link: ", bcolors.OKBLUE + link + bcolors.ENDC, "\nStatus:", bcolors.OKGREEN + status + bcolors.ENDC) c.execute( '''INSERT INTO onionsites (url, title, status, status_code) VALUES (?, ?, ?, ?);''', (str(link), str(siteTitle), str(status), int(response_code))) # Storing all the results conn.commit() elif response_code == 500: status = 'Failed' print("Status code:", response_code) print("Link: ", bcolors.FAIL + str(link) + bcolors.ENDC, "\nStatus:", bcolors.FAIL + status + bcolors.ENDC)
def checkIPs(): """ This funciton checks first the Public IP and then the TOR IP and prints them out. """ proxy = SOCKSProxyManager('socks5h://localhost:9050/') http = urllib3.PoolManager() myip = http.request('GET', 'http://httpbin.org/ip') try: global torIP torIP = proxy.request('GET', 'http://httpbin.org/ip') except NameError: print( bcolors.FAIL + "proxy isn't running or address:port is wrong.\nScript will exit." + bcolors.ENDC) exit(1) jsonIP = json.loads(myip.data.decode('utf-8')) jsonTorIP = json.loads(torIP.data.decode('utf-8')) print("Your real IP Address is: " + bcolors.OKBLUE + jsonIP["origin"] + bcolors.ENDC) print("Your Tor IP Address is: " + bcolors.FAIL + jsonTorIP["origin"] + bcolors.ENDC) print(bcolors.FAIL + "=" * 50 + bcolors.ENDC)
def resolveJson(path): headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36' } proxy = SOCKSProxyManager('socks5://localhost:1080/', headers=headers) response = proxy.request( 'get', "https://storage.googleapis.com/fantasylauncer/wallpaper/wallpaperOnline.json" ) with open("wallpaperOnline.json", 'wb+') as f: f.write(response.data) f.close()
def saver(url): mgr = SOCKSProxyManager( 'socks5://%s:%s/' % (config.get('Proxy', 'host'), config.get('Proxy', 'port')), **{ 'username': config.get('Proxy', 'username'), 'password': config.get('Proxy', 'password') }) tordata = mgr.request('GET', url, headers={ 'connection': 'keep-alive' }).data out_dir = config.get('Torrent', 'save_dir') with open(os.path.join(out_dir, os.path.basename(url)), 'wb') as torfile: torfile.write(tordata) return tordata
def get_data(uri, instrument_id, num): unixtime = int(time.time()) sign_str = '%d%s%s%s' % (unixtime, method, uri, secretkey) # 签名 sha256 = hashlib.sha256( sign_str.encode('utf-8')).hexdigest().encode('utf-8') signature_str = base64.b64encode(sha256).decode('utf-8') #print(sign_str.encode('utf-8')) #print(sha256) #print(signature_str) headers = { 'OK-ACCESS-KEY': apikey, 'OK-ACCESS-SIGN': signature_str, 'OK-ACCESS-TIMESTAMP': '%d' % unixtime, 'OK-ACCESS-PASSPHRASE': Passphrase, 'Accept': 'application/json', 'Content-Type': 'application/json; charset=UTF-8', } # 发起调用 url = host + uri print(url) start_time = datetime.now() # 不使用代理 #pool = urllib3.PoolManager(num_pools=2, timeout=180, retries=False) #r = pool.urlopen(method, url, headers=headers) # 使用sock5代理 proxy = SOCKSProxyManager('socks5h://localhost:1080/') r = proxy.request(method, url, headers=headers) print('[Time taken: {!s}]'.format(datetime.now() - start_time)) #print(r.status) #print(r.data) if r.status != 200: print("fail: ", r.status, r.data) return [] # 获取数据,截断为num, 注意顺序:最新数据在前 data = json.loads(r.data.decode('utf-8')) data = data[:num] return data
class TorProxy: def __init__(self): self.proxy = SOCKSProxyManager('socks5://127.0.0.1:9050') def new_ip(self): with Controller.from_port(port=9051) as controller: controller.authenticate(password="******") controller.signal(Signal.NEWNYM) controller.close() def request(self, url): self.new_ip() headers = {'User-Agent': UserAgent().random} rq = self.proxy.request('GET', url, headers=headers) return rq
def check_proxy(proxy, proxy_type): if proxy_type == 'socks': pool = SOCKSProxyManager('socks5://%s' % proxy) else: pool = urllib3.ProxyManager('http://%s' % proxy) retries = urllib3.Retry(total=None, connect=False, read=False, redirect=10, raise_on_redirect=False) timeout = urllib3.Timeout(connect=CONNECT_TIMEOUT, read=READ_TIMEOUT) op = { 'status': None, 'connect_time': None, 'read_time': None, 'error': None, } try: start_time = time.time() res = pool.request('GET', 'https://en.wikipedia.org/robots.txt', retries=retries, timeout=timeout, preload_content=False) connected_time = time.time() op['connect_time'] = round(connected_time - start_time, 2) data = res.read() op['read_time'] = round(time.time() - connected_time, 2) except Exception as ex: error = type(ex).__name__ op['error'] = error if error in ('NewConnectionError', 'ConnectTimeoutError'): op['status'] = 'connect_fail' elif error in ( 'ProtocolError', 'ReadTimeoutError', ): op['status'] = 'read_fail' else: raise Exception('Unexpected error: %s' % error) else: if b'# Wikipedia work bots:' in data: op['status'] = 'ok' else: op['status'] = 'data_fail' return op
def pageRequest(url): global roundRobin proxy = SOCKSProxyManager('socks5://localhost:'+str(torPort), cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), headers={'user-agent': randomUserAgent(), 'Cookie': ''}) http = urllib3.PoolManager( 1, cert_reqs='CERT_REQUIRED', ca_certs=certifi.where(), headers={'user-agent': randomUserAgent(), 'Cookie': ''}) if roundRobin % 2: response = http.request('GET', url) else: if torSupport: response = proxy.request('GET', url) else: response = http.request('GET', url) roundRobin += 1 if not roundRobin % 60: newTorIdentity() return response.data
def query(url, sport): """Uses urllib3 to fetch a site using the proxy on the @sport. """ proxy = SOCKSProxyManager('socks5://localhost:{}/'.format(sport)) r = proxy.request('GET', url) return r.data
def prepare_options(self): if is_blank(self._url): raise Exception("GoogleArtsCrawlerOption , url is blank!") uprs = parse_url(url=self._url) if not uprs.host == 'artsandculture.google.com': raise Exception("GoogleArtsCrawlerOption, url netloc is not `artsandculture.google.com`") self._url = "https://{0}{1}".format(uprs.host, uprs.path) # download webdriver if self._webdriver_execute_path is None and self._need_download_webdrive: default_webdrive_path = "webdriver" if os.path.isdir(default_webdrive_path): default_webdrive_files = os.listdir(default_webdrive_path) if len(default_webdrive_files) > 0: default_webdrive_execute_file = os.path.join(default_webdrive_path, default_webdrive_files[0]) if os.path.isfile(default_webdrive_execute_file): print("==> webdriver has exist at {0}".format(default_webdrive_execute_file)) self._webdriver_execute_path = default_webdrive_execute_file if self._webdriver_execute_path is None: if WINDOWS: os_name = "Windows" webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_win32.zip" elif DARWIN: os_name = "Mac OS" webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_mac64.zip" elif LINUX: os_name = "Linux" webdriver_download_url = "http://chromedriver.storage.googleapis.com/78.0.3904.70/chromedriver_linux64.zip" else: raise Exception("GoogleArtsCrawlerOptions, unknown platform !") print("==> current operation system : {0}".format(os_name)) print("==> prepare download webdriver : {0}".format(webdriver_download_url)) default_download_tmp = "tmp" webdriver_zip_filename = webdriver_download_url.split("/")[-1] webdriver_local_zip_filepath = os.path.join(default_download_tmp, webdriver_zip_filename) # not exist if not os.path.isfile(webdriver_local_zip_filepath): http = SOCKSProxyManager('socks5://localhost:1086/') # http = PoolManager() response = http.request('GET', webdriver_download_url, preload_content=False) if not os.path.isdir(default_download_tmp): os.mkdir(default_download_tmp) with open(webdriver_local_zip_filepath, mode="wb") as fd: while True: data = response.read(1024) if not data: break fd.write(data) response.release_conn() print("==> webdriver zip file download finished , location at : {0}".format( os.path.abspath(webdriver_local_zip_filepath))) else: print("==> webdriver zip file has existed at {0}".format(webdriver_local_zip_filepath)) with ZipFile(webdriver_local_zip_filepath, 'r') as zipfile: zipfile.extractall(path=default_webdrive_path) if self._need_clear_cache: shutil.rmtree(default_download_tmp) self._webdriver_execute_path = os.path.join(default_webdrive_path, os.listdir(default_webdrive_path)[0]) if is_blank(self._webdriver_execute_path): raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is blank!") if not os.path.isfile(self._webdriver_execute_path): raise Exception("GoogleArtsCrawlerOption , webdriver_execute_path is not exist, this is file!") if LINUX or DARWIN: os.chmod(self._webdriver_execute_path, 0o777) # self._chrome_options.binary_location = os.path.abspath(self._webdriver_execute_path) mobile_emulation = { "deviceMetrics": {"width": self._size, "height": self._size, "pixelRatio": 1.0}, "userAgent": "Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 " "(KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19"} self._chrome_options.add_experimental_option("mobileEmulation", mobile_emulation) self._chrome_options.add_argument('--no-sandbox') self._chrome_options.add_argument('--disable-dev-shm-usage') self._chrome_options.add_argument('--disable-gpu') self._chrome_options.add_argument("--disable-dev-shm-usage") self._chrome_options.add_argument("start-maximized") self._chrome_options.add_argument("disable-infobars") self._chrome_options.add_argument("--disable-extensions") if not self._is_debug: self._chrome_options.add_argument("--headless") self._output_path = DEFAULT_GCO_OUTPUT_PATH if self._output_path is None else self._output_path self._size = DEFAULT_GCO_SIZE if self._size is None or self._size < 1 else self._size self._init_delay_time = DEFAULT_GCO_INIT_DELAY if self._init_delay_time is None or self._init_delay_time < 1 else self._init_delay_time if not os.path.isdir(self._output_path): os.makedirs(self._output_path) if not os.path.isdir(self._partial_tmp_path): os.makedirs(self._partial_tmp_path) if self._is_debug: print("GoogleArtsCrawlerOptions:") print("==> url:{0}".format(self._url)) print("==> webdriver_execute_path:{0}".format(os.path.abspath(self._webdriver_execute_path))) print("==> output :{0}".format(os.path.abspath(self._output_path))) return self
def test_socks_proxy(proxy,time): prs = SOCKSProxyManager('socks5://'+proxy) prs.request('GET', 'https://www.youtube.com', timeout=time)
class Request(Singleton): __request_timeout = 180 __allow_redirects = False __default_user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0' def __parse_custom_header(self, custom_header): parsed_custom_header = custom_header.split(':') custom_header_key = parsed_custom_header[0].strip() custom_header_value = parsed_custom_header[1].strip() self.__headers[custom_header_key] = custom_header_value def __init__(self, url, user_agent, cookies_string=False, custom_header=False, insecure_ssl='false', proxy=False): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) self.__url = url self.__headers = dict() self.__headers[ 'User-Agent'] = self.__default_user_agent if user_agent == 'default' else user_agent if cookies_string: self.__headers['Cookie'] = cookies_string if custom_header: self.__parse_custom_header(custom_header) self.__verify = 'CERT_REQUIRED' if insecure_ssl == 'false' else 'CERT_NONE' if proxy: proxy_type = proxy.split('://')[0] if proxy_type == 'http' or proxy_type == 'https': self.__request_obj = urllib3.ProxyManager( proxy, ssl_version=ssl.PROTOCOL_TLSv1, timeout=self.__request_timeout, cert_reqs=self.__verify) else: self.__request_obj = SOCKSProxyManager( proxy, ssl_version=ssl.PROTOCOL_TLSv1, timeout=self.__request_timeout, cert_reqs=self.__verify) else: self.__request_obj = urllib3.PoolManager( ssl_version=ssl.PROTOCOL_TLSv1, timeout=self.__request_timeout, cert_reqs=self.__verify) # print (vars(self)) def send_request(self, request): try: response_object = self.__request_obj.request( 'POST', self.__url, fields={'data': request}, redirect=self.__allow_redirects, headers=self.__headers, retries=False) except KeyboardInterrupt: raise Exception('Keyboard interrupt issued') return response_object.status, response_object.headers, response_object.data
def test_socks_proxy(proxy): pr = SOCKSProxyManager('socks5://'+proxy) pr.request('GET', 'https://www.youtube.com')