def __init__(self, _=None): """Build self._urllib_opener""" proxy = super().__str__() if proxy == "None": self._urllib_opener = build_opener() return components = list(re.match(self._match_regexp, proxy).groups()) self.scheme, self.host, self.port = components self.components = components if self.scheme == "socks4": socks4_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS4, self.host, int(self.port)) self._urllib_opener = build_opener(socks4_handler) elif self.scheme == "socks5": socks5_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, self.host, int(self.port)) self._urllib_opener = build_opener(socks5_handler) else: proxy_handler = ProxyHandler({'http': proxy, 'https': proxy}) self._urllib_opener = build_opener(proxy_handler)
def _create_opener(self): opener = urllibrequest.build_opener() if self.useProxy: if self.proxy_protocol == "socks4": opener = urllibrequest.build_opener(SocksiPyHandler(socks.SOCKS4, self.proxy_host, self.proxy_port, username=self.proxy_username, password=self.proxy_password)) if self.proxy_protocol == "socks5": opener = urllibrequest.build_opener(SocksiPyHandler(socks.SOCKS5, self.proxy_host, self.proxy_port, username=self.proxy_username, password=self.proxy_password)) return opener
def open(self, url, data=None, headers=None, cookies=None, timeout=60): self.log.debug('%s request to %s\nHeaders: %s\nTimeout: %s', 'GET' if data else 'POST', url, headers, timeout) cookies = cookies or http.cookiejar.LWPCookieJar() opener = urllib.request.build_opener( SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, self.host, self.socks_port), urllib.request.HTTPCookieProcessor(cookies)) status_code = None if headers: opener.addheaders = [item for item in list(headers.items())] try: response = opener.open(url, data, timeout) except http.client.IncompleteRead as e: response = e.partial except Exception as e: if hasattr(e, 'code'): status_code = e.code self.log.debug('Failed to open %s, %s', url, e) return Response(None, None, cookies) try: text = response.read() except Exception as e: text = None self.log.debug('Unable to read response content: %r', e) try: status_code = status_code or response.getcode() except Exception as e: status_code = None self.log.debug('Error during getting response status code: %r', e) return Response(text, status_code, cookies)
def check_proxy(proxy, p=False): """Checks the status of the proxy object. Args: proxy: A proxy to check. p: A boolean variable that indicates whether to print or return the value. Returns: The status of the proxy. If there was a failure, the reason is given back to the caller as an exception string. """ if isinstance(proxy, dict): proxy = Proxy(proto=proxy['proto'], host=proxy['host'], port=proxy['port'], username=proxy['username'], password=proxy['password']) try: opener = urllib2.build_opener( SocksiPyHandler(scheme_mapping[proxy.proto], proxy.host, int(proxy.port))) opener.open("http://httpbin.org/ip").read() except socks.ProxyConnectionError as e: if p: print '{} is \033[91m offline \033[0m'.format(proxy) else: return ('offline', str(e)) else: if p: print '{} is \033[91m online \033[0m'.format(proxy) else: return 'online'
def api(self, command, args=None): """ Main API function. Returns: returns 'False' if invalid command or if no APIKey or Secret is specified (if command is "private"). returns {"error":"<error message>"} if API error. """ logging.info("Poloniex command: {}, args: {}".format(command, args)) args = args or {} if command in PUBLIC_COMMANDS: url = 'https://poloniex.com/public?' args['command'] = command # prevent urllib from complaining when using a proxy context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE opener = build_opener( SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, constants.PROXY_ADDR, constants.PROXY_PORT, True, context=context)) ret = opener.open(Request(url + urlencode(args))) return json.loads(ret.read().decode(encoding='UTF-8')) else: return False
def down_web_page_html(url, headers=None, proxy=None, retry=3, data=None): print('下载如下链接:', url) # headers = {'User-agent': user_agent} request = urllib.request.Request( url, data=urllib.parse.urlencode(data).encode('utf-8'), headers=headers) handler = urllib.request.HTTPCookieProcessor() opener = urllib.request.build_opener(handler) if proxy: # proxy_params = {urllib.parse.urlparse(url).scheme: proxy} proxy_params = {'https': proxy} # opener.add_handler(urllib.request.ProxyHandler(proxy_params)) opener.add_handler(SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080)) try: response = opener.open(request) html = response.read() code = response.code except urllib.request.URLError as e: print('下载遇到错误:', e.reason) html = '' if hasattr(e, 'code'): code = e.code if retry > 0 and 400 <= code <= 600: retry -= 1 time.sleep(3) WebPageDown.down_web_page_html(url, headers, proxy, retry - 1) else: code = None return {'html': html, 'code': code}
def Test_SS_speed(self, ss_list): result_list = [] os.system('pkill sslocal') for i in range(len(ss_list)): server_port = "108" + str(i) command = "sslocal -s " + "\"" + ss_list[i][ 'server'] + "\"" + " -p " + "\"" + ss_list[i][ 'server_port'] + "\"" + " -l " + server_port + " -k " + "\"" + ss_list[ i]['password'] + "\"" + " -m " + "\"" + ss_list[i][ 'method'] + "\"" + " 2>1 &" # print command os.system(command) time.sleep(2) opener = urllib2.build_opener( SocksiPyHandler(socks.SOCKS5, "127.0.0.1", int(server_port))) start_time = time.time() try: if opener.open('https://www.tumblr.com/').getcode() == 200: end_time = time.time() speed = end_time - start_time else: speed = 10000000 except Exception: speed = 10000000 pass ss_list[i]['speed'] = speed result_list.append(ss_list[i]) time.sleep(1) os.system("pkill sslocal") return result_list
def download_image(url, save_name, proxy=False): try: fp = '' if proxy is True: opener = urllib2.build_opener( SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 9999)) fp = opener.open(url, timeout=40) else: fp = urllib2.urlopen(url, timeout=40) data = fp.read() fp.close() print url + ' downloading...' fid = open(save_name, 'w+b') fid.write(data) flag = True flsize = os.path.getsize(save_name) print 'file size:%d ' % flsize if flsize < 10: flag = False ###### except Exception: print url + ' downloading io error... ', sys.exc_info()[0] flag = False return flag
def checkproxy(port, type): if type == 'http': handler = socks.HTTP elif type == 'socks': handler = socks.SOCKS5 elif type == 'http0': proxy = urllib.request.ProxyHandler( {'http': PROXYHOST + ':' + str(port)}) else: print("[unsupported type]") return if type == 'http0': opener = urllib.request.build_opener(proxy) else: opener = urllib.request.build_opener( SocksiPyHandler(handler, PROXYHOST, port)) try: response = opener.open(TESTURI, timeout=TIMEOUT).read() except Exception as e: print("[%5s]" % ('dead')) return print("[%5s]" % ('alive'))
def AuthorizeTokens(client_id, client_secret, authorization_code, redirect_uri, google_account_base_url): """Obtains OAuth access token and refresh token. This uses the application portion of the "OAuth2 for Installed Applications" flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse Args: client_id: Client ID obtained by registering your app. client_secret: Client secret obtained by registering your app. authorization_code: code generated by Google Accounts after user grants permission. Returns: The decoded response from the Google Accounts server, as a dict. Expected fields include 'access_token', 'expires_in', and 'refresh_token'. """ params = {} params['client_id'] = client_id params['client_secret'] = client_secret params['code'] = authorization_code params['redirect_uri'] = redirect_uri params['grant_type'] = 'authorization_code' request_url = AccountsUrl(command='o/oauth2/token', google_account_base_url=google_account_base_url) print request_url, redirect_uri from sockshandler import SocksiPyHandler import socks opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, "localhost", proxyport=1080)) response = opener.open(request_url, urllib.urlencode(params)).read() print response return json.loads(response)
def test_socks_connection(url, proxy_host, proxy_port): """Return HTTP code from opening URL via SOCKS proxy""" opener = urllib.request.build_opener( SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, proxy_host, proxy_port)) resp = opener.open(url) try: return resp.code finally: resp.close()
def get_page(ResRefererEx): global PK_status global htmlshash # Use a proxy if declared in config file try: # Use a HTTP proxy if proxy_type in 'http': http_proxy proxy_support = urllib.request.ProxyHandler({'http': http_proxy}) opener = urllib.request.build_opener(proxy_support) urllib.request.install_opener(opener) # Use a SOCKS5 proxy elif proxy_type in 'socks': socks_proxy_server socks_proxy_port opener = urllib.request.build_opener( SocksiPyHandler(socks.SOCKS5, socks_proxy_server, int(socks_proxy_port), True)) urllib.request.install_opener(opener) except NameError: pass try: request = urllib.request.Request( url=ResRefererEx, # Force user-agent headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36' }) response = urllib.request.urlopen(request, timeout=5) resp_code = response.getcode() htmldata = str(response.read().decode('utf-8')) if resp_code == 200: try: # If page contains tracking_file_request if RegRequest2.finditer(htmldata): PK_status = 'UP' # Create SHA256 hash of HTML page content htmlshash = hashlib.sha256( htmldata.encode('utf-8')).hexdigest() else: PK_status = 'Probably removed' except: err = sys.exc_info() print(err) pass else: PK_status = 'DOWN' except: #err = sys.exc_info()[1] err = sys.exc_info()[1] PK_status = ('can\'t connect (' + str(err) + ')') pass
def check_proxy(pip, timeout): try: opener = urllib.request.build_opener( SocksiPyHandler(socks.SOCKS5, pip[0:pip.find(":")], int(pip[pip.find(":") + 1:len(pip)]))) opener.open("https://api.ipify.org", timeout=timeout) proxys.add(pip) except Exception as e: badproxys.add(pip) return proxys
def request_webpage_via_ss_proxy(url, proxy): """ 使用SOCKS5代理进行http请求 proxy[0] 代理IP proxy[1] 代理端口 """ import socks from sockshandler import SocksiPyHandler opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, proxy[0], int(proxy[1]))) return opener.open(url, timeout=6)
def build_urllib2_proxy_handler(url): handler = None if url: m = re.match( r'^(?P<type>\w+)://(?P<ip>\d+\.\d+\.\d+\.\d+)(:(?P<port>\d+))?', url) if m and m.group('type').startswith('socks5'): handler = SocksiPyHandler( socks.SOCKS5, m.group('ip'), int(m.group('port')) if m.group('port') else 1080) return handler
def test_server(port): opener = urllib2.build_opener(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, 'localhost', port)) try: st = time.time() content = opener.open("https://www.youtube.com", timeout=10).read() if not content: return -1 else: return time.time() - st except: return -1
def test_access_2(): import urllib import socks from sockshandler import SocksiPyHandler try: opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080)) x = opener.open("http://www.google.com/", timeout=5) return x.getcode() except: return -1
def __init__(self, port_offset): self.port_offset = int(port_offset) if not 0 <= self.port_offset < 1000000: raise Error('Invalid port_offset %s' % self.port_offset) self.socks_port = base_socks_port + self.port_offset self.control_port = base_control_port + self.port_offset self.path = tors_dir + "/" + str(self.port_offset) self.opener = None makedir(self.path) self.opener = urllib2.build_opener( SocksiPyHandler(socks.SOCKS5, "127.0.0.1", self.socks_port))
def change_proxy(): if USE_PROXY and PROXY_PORTS: print("Changing proxy..") port = random.choice(PROXY_PORTS) debug("Using proxy port #%d" % port) socks_handler = SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, PROXY_HOST, port) opener = urllib.request.build_opener(socks_handler) urllib.request.install_opener(opener) return port return None
def latency(url): """测试三个外网的get的平均延时""" t_start = time.time() try: opener = urllib2.build_opener( SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080)) time.sleep(2) s = opener.open(url) # print s.read() t_end = time.time() return t_end - t_start except: return 9999999
def getJsonReponse(tweetCriteria, refreshCursor, cookieJar): url = "https://twitter.com/i/search/timeline?f=realtime&q=%s&src=typd&max_position=%s" urlGetData = '' if hasattr(tweetCriteria, 'querySearch'): urlGetData += tweetCriteria.querySearch if hasattr(tweetCriteria, 'username'): urlGetData += ' from:' + tweetCriteria.username if hasattr(tweetCriteria, 'since'): urlGetData += ' since:' + tweetCriteria.since if hasattr(tweetCriteria, 'until'): urlGetData += ' until:' + tweetCriteria.until if hasattr(tweetCriteria, 'topTweets'): if tweetCriteria.topTweets: url = "https://twitter.com/i/search/tweets?q=%s&src=typd&max_position=%s" url = url % (urllib.quote(urlGetData), refreshCursor) headers = [('Host', "twitter.com"), ('User-Agent', "Mozilla/6.0 (Windows NT 6.1; Win64; x64)"), ('Accept', "application/json, text/javascript, */*; q=0.01"), ('Accept-Language', "de,en-US;q=0.7,en;q=0.3"), ('X-Requested-With', "XMLHttpRequest"), ('Referer', url), ('Connection', "keep-alive")] opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(cookieJar), SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080)) opener.addheaders = headers try: response = opener.open(url) jsonResponse = response.read() except Exception as e: if e == KeyboardInterrupt: sys.exit() else: return None try: dataJson = json.loads(jsonResponse) except: return [] return dataJson
def attack2(host1, port): try: opener = urllib2.build_opener( SocksiPyHandler(socks.SOCKS5, "127.0.0.1", port, True)) opener.addheaders = [('User-Agent', random.choice(useragents))] opener.addheaders = [('Cache-Control', 'no-cache')] opener.addheaders = [('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.7')] opener.addheaders = [('Referer', random.choice(referers) + str(randstr))] opener.addheaders = [('Keep-Alive', random.randint(110, 120))] opener.addheaders = [('Connection', 'keep-alive')] opener.addheaders = [('Host', host1)] except: print(' Error2')
def __setProxy4Urllib(self): if self.proxy_flag is False and self.__proxyConfigExists(): # set proxy for urllib2 self.log.info('setting proxy %s %s:%s', protocol, server, port) from sockshandler import SocksiPyHandler global proxy proxy = SocksiPyHandler(protocol, server, int(port)) opener = urllib2.build_opener(proxy) urllib2.install_opener(opener) self.proxy_flag = True else: self.log.info( 'Proxy has bean configured: protocol=%d server=%s port=%s', protocol, server, port)
def test_proxy_by_urllib2(): import urllib2 import socks from sockshandler import SocksiPyHandler headers = {'Host': 'app-api.pixiv.net', 'User-Agent': 'PixivIOSApp/6.0.9', 'Accept-Language': 'zh', 'Authorization': 'Bearer %s' % 'uMzgoLH6TysDAvu8594IvustucSHz0hwThlMYG3uHY0', 'Accept-Encoding': 'gzip'} httpHandler = urllib2.HTTPHandler(debuglevel=1) httpsHandler = urllib2.HTTPSHandler(debuglevel=1) opener = urllib2.build_opener(httpHandler, httpsHandler, SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 1080)) urllib2.install_opener(opener) req = urllib2.Request("https://app-api.pixiv.net/v1/illust/detail?illust_id=55418", headers=headers) x = urllib2.urlopen(req) print (x.read())
def crawler(self, counter): ip = '' port = 1080 url = '' filename = "images/MJ%d.jpg" % counter opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, ip, port)) opener.addheaders = [( 'User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36' )] response = opener.open(url) htmlData = response.read() f = open(filename, 'w') f.write(htmlData) f.close() return filename
def install_proxy(host, port, proxy_type=socks.HTTP): """ :param host: proxy host :param port: proxy ip :param proxy_type: proxy type SOCKS4 = 1 SOCKS5 = 2 HTTP = 3 """ if proxy_type not in (socks.SOCKS5, socks.SOCKS4, socks.HTTP): raise RuntimeError("Not support proxy type") proxy = urllib2.ProxyHandler({'http': host + ":" + port}) if proxy_type == 1 \ else SocksiPyHandler(proxy_type, host, port) opener = urllib2.build_opener(urllib2.HTTPHandler(debuglevel=1), urllib2.HTTPSHandler(debuglevel=1), proxy) urllib2.install_opener(opener)
def __loadCookies(self): if self.alt_cookiejar is not None: self.alt_cookiejar.init_agent(new_headers=self.browserHeaders) self.cj = self.alt_cookiejar else: self.cj = http.cookiejar.LWPCookieJar( ) # This is a subclass of FileCookieJar # that has useful load and save methods if self.cj is not None: if os.path.isfile(self.COOKIEFILE): try: self.__updateCookiesFromFile() # self.log.info("Loading CookieJar") except: self.log.critical("Cookie file is corrupt/damaged?") try: os.remove(self.COOKIEFILE) except FileNotFoundError: pass if http.cookiejar is not None: # self.log.info("Installing CookieJar") self.log.debug(self.cj) cookieHandler = urllib.request.HTTPCookieProcessor(self.cj) args = (cookieHandler, Handlers.HTTPRedirectHandler) if self.credHandler: print("Have cred handler. Building opener using it") args += (self.credHandler, ) if self.use_socks: print("Using Socks handler") if not HAVE_SOCKS: raise RuntimeError( "SOCKS Use specified, and no socks installed!") args = (SocksiPyHandler(socks.SOCKS5, "127.0.0.1", 9050), ) + args self.opener = urllib.request.build_opener(*args) #self.opener.addheaders = [('User-Agent', 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)')] self.opener.addheaders = self.browserHeaders #urllib2.install_opener(self.opener) for cookie in self.cj: self.log.debug(cookie)
def parse_proxy_url(proxy_url): proto = None host = None port = None login = None password = None if not proxy_url: return proto_parsed = proxy_url.split("://")[0].lower() if proto_parsed == "socks5": proto = socks.SOCKS5 elif proto_parsed == "socks4": proto = socks.SOCKS4 elif proto_parsed == "http": proto = socks.HTTP elif proto_parsed == "https": proto = socks.HTTP host_string = proxy_url.split("://")[1] if '@' in host_string: ary = host_string.split("@") user_string = ary[0] host_string = ary[1] ary = user_string.split(":") login = ary[0] password = ary[1] if host_string: ary = host_string.split(":") host = ary[0] port = ary[1] return SocksiPyHandler(proxytype=proto, proxyaddr=host, proxyport=int(port), username=login, password=password, rdns=True)
def get_opener(config): proxy = config.get('proxy') if not proxy: return urllib2.build_opener() url = urlparse.urlparse(proxy) if url.scheme == 'socks5': m = re.match(r'^([^:]+):([^@]+)@([^:]+):(.+)$', url.netloc) if m is None: print('Proxy URL should look like this: socks5://user:password@host:port', file=sys.stderr) exit(2) user, passwd, host, port = m.groups() try: from sockshandler import SocksiPyHandler opener = urllib2.build_opener(SocksiPyHandler(socks.SOCKS5, host, int(port), True, user, passwd)) return opener except ImportError: print('Install PySocks to use a socks5 proxy.', file=sys.stderr) exit(5) print('Unsupported proxy type: %s' % url.scheme, file=sys.stderr) exit(3)
def get_url_title(url): enc = ['utf8', 'iso-8869-1', 'shift-jis'] title = '' headers = { 'User-Agent': USER_AGENT } try: req = urllib.request.Request(url, data=None, headers=headers) host = urllib.request.urlparse(url).netloc.split(':')[0] if PROXY_HOST and PROXY_PORT and re.match(PROXY_REGEX, host): opener = urllib.request.build_opener(SocksiPyHandler(socks.SOCKS5, PROXY_HOST, PROXY_PORT)) resp = opener.open(req, timeout=5) else: resp = urllib.request.urlopen(req, timeout=5) except urllib.error.HTTPError as e: return 'Status: ' + str(e.code) except urllib.error.URLError as e: return 'Error: ' + str(e.reason) except Exception as e: return # get the site's title, only in html content if 'content-type' in resp.headers and 'html' in resp.headers['content-type']: # try the charset set in the html header first, if there is one if 'charset=' in resp.headers['content-type']: enc = enc + [resp.headers['content-type'].split('charset=')[-1]] # read up to 1mb chunk = resp.read(1024 * 1024) parser = TitleParser() for e in enc: try: decoded_chunk = chunk.decode(e, 'ignore') parser.feed(decoded_chunk) if parser.done: title = parser.title parser.close() if len(title) > 0: esc = parser.unescape(title) return 'Title: ' + esc.strip() except Exception as ex: pass # no title, try to output some other useful data info = [] if 'content-type' in resp.headers: info.append('Type: ' + resp.headers['content-type'].split(';')[0]) if 'content-length' in resp.headers: info.append('Size: ' + humanify(int(resp.headers['content-length']))) if 'last-modified' in resp.headers: d = resp.headers['last-modified'] try: parsed_date = dateutil.parser.parse(d) d = parsed_date.strftime('%F %T') + ' ' + parsed_date.tzname() except ValueError: pass info.append('Modified: ' + d) return ', '.join(info)