if os.name == 'nt': import ctypes ctypes.windll.kernel32.SetConsoleTitleW('%s %s' % (_name, __version__)) config = LoadConfig(CONFIG) logger = logging.getLogger(__name__) logger.setLevel(getattr(logging, config.LOGLEVEL, logging.INFO)) handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s %(message)s', datefmt='[%H:%M]') handler.setFormatter(formatter) logger.addHandler(handler) pools = ConnectionPools(CONFIG) proxpool = urllib3.ProxyManager(config.PROXADDR, num_pools=10, maxsize=8, # A little longer than timeout of rear pool # to avoid trigger front server exception handler timeout=urllib3.util.timeout.Timeout(connect=90.0, read=310.0)) frontserver = FrontServer(('', config.FRONTPORT), FrontRequestHandler) rearserver = RearServer(('', config.REARPORT), RearRequestHandler) for worker in (frontserver.serve_forever, rearserver.serve_forever, pools.reloadConfig): thread = threading.Thread(target=worker) thread.daemon = True thread.start() print("=" * 76) print('%s %s (urllib3/%s)' % (_name, __version__, urllib3.__version__)) print() print(' FrontServer : localhost:%s' % config.FRONTPORT) print(' RearServer : localhost:%s' % config.REARPORT)
def __init__(self, pools_size=4, max_size=4): # urllib3.PoolManager will pass all kw parameters to connectionpool # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/poolmanager.py#L75 # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/connectionpool.py#L680 # ca_certs vs cert_file vs key_file # http://stackoverflow.com/a/23957365/2985775 # cert_reqs if Configuration().verify_ssl: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE # ca_certs if Configuration().ssl_ca_cert: ca_certs = Configuration().ssl_ca_cert else: # if not set certificate file, use Mozilla's root certificates. ca_certs = certifi.where() # cert_file cert_file = Configuration().cert_file # key file key_file = Configuration().key_file # proxy proxy = Configuration().proxy proxy_username = Configuration().proxy_username proxy_password = Configuration().proxy_password retries = urllib3.util.Retry() retries.allowed_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS', 'POST', 'PUT', 'PATCH', 'TRACE'} # https pool manager if proxy: headers = None if proxy_username and proxy_password: headers = urllib3.make_headers(proxy_basic_auth=proxy_username + ':' + proxy_password) self.pool_manager = urllib3.ProxyManager( retries=retries, num_pools=pools_size, maxsize=max_size, block=True, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=cert_file, key_file=key_file, proxy_url=proxy, proxy_headers=headers ) else: self.pool_manager = urllib3.PoolManager( retries=retries, num_pools=pools_size, maxsize=max_size, block=True, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=cert_file, key_file=key_file )
import os import MongoDB from PIL import Image from urllib3.contrib.socks import SOCKSProxyManager #just a test url = "https://game.nihaoma.top/t1/?template=blue&token=b595fa57&CateID=10#/" http = urllib3.PoolManager( cert_reqs = 'CERT_REQUIRED', ca_certs = certifi.where() ) ######## 访问某些网站使用本地ssr代理 proxy = urllib3.ProxyManager('http://127.0.0.1:1087', 'https://127.0.0.1:1087') sockproxy = SOCKSProxyManager('socks5://localhost:1086') ''' 通过ssr代理访问 端口1080 无Referer ''' # def visitByProxy(url): # try: # web = proxy.request('GET', url, # headers={ # 'User-Agent': # 'ozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15'
url:设置请求的链接 fields:设置对应请求需要发送的参数,该数据时字典类型,可以通过键值对设置 headers:用来设置请求的头部信息,比如设置数据的编码格式,数据的参数类型,模拟浏览器发送请求等 """ header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36', 'Content-type': 'text/json'} # 建立ip代理池 proxy_list = ["HTTP://103.10.86.203:8080", "HTTP://110.16.80.106:8080", "HTTP://27.128.187.22:3128", "HTTP://183.185.1.47:9797", "HTTP://180.101.99.110:80", "HTTP://175.150.107.61:1133", "HTTP://120.77.206.107:3128", "HTTP://119.131.88.242:9797", "HTTP://125.88.190.1:3128", "HTTP://171.37.79.169:9797", "HTTP://59.57.148.165:9999", "HTTP://183.164.238.74:9999", "HTTP://117.69.200.167:9999", "HTTP://180.101.99.110:80", "HTTP://1.196.161.203:9999", "HTTP://103.10.86.203:8080", "HTTP://27.128.187.22:3128", "HTTP://117.57.90.81:9999", "HTTP://183.185.1.47:9797", "HTTP://117.28.96.18:9999"] # 选择随机IP ip = random.choice(proxy_list) # 简单get请求 # http = urllib3.PoolManager(headers=header) # 设置IP代理(使用IP代理爬取数据) http = urllib3.ProxyManager(ip, proxy_headers=header) """ timeout=urllib3.Timeout(connect=3, read=3):将连接的timeout和读的timeout分开设置。如果想让所有的request都遵循一个timeout,可以将timeout参数定义在PoolManager中 retries=urllib3.Retry(3, redirect=2):进行3次请求重试,但是只进行2次重定向。如果想让所有请求都遵循一个retry策略,可以在PoolManager中定义retry参数 """ response = http.request( 'get', 'http://www.baidu.com', timeout=urllib3.Timeout(connect=3, read=2), retries=urllib3.Retry(1, redirect=2)) if response.status == 200: print(ip) # 模拟浏览器发送请求
def send_request(self, method, target_url, preload_content=True, query_param=None, body_param=None, enc='utf-8', cert_ignore=False): res_header = '' res_body = '' server_header = '-' res = None content_type_value = '' # Initialize empty parameter set. if query_param is None: query_param = {} if body_param is None: body_param = {} # Set proxy server. http = None ctx = ssl.create_default_context() ctx.set_ciphers('DEFAULT') # Ignore verification of certificate. if cert_ignore: ctx.check_hostname = False urllib3.disable_warnings() # ctx.set_ciphers('DEFAULT@SECLEVEL=1') if self.proxy != '': self.print_message(WARNING, 'Set proxy server: {}'.format(self.proxy)) if self.proxy_user != '': headers = urllib3.make_headers( proxy_basic_auth=self.proxy_user + ':' + self.proxy_pass) http = urllib3.ProxyManager(timeout=self.con_timeout, headers=self.http_req_header, proxy_url=self.proxy, proxy_headers=headers) else: http = urllib3.ProxyManager(timeout=self.con_timeout, headers=self.http_req_header, proxy_url=self.proxy) else: if cert_ignore: http = urllib3.PoolManager(timeout=self.con_timeout, headers=self.http_req_header, ssl_version=ssl.PROTOCOL_TLSv1, ssl_context=ctx, cert_reqs=ssl.CERT_NONE) else: http = urllib3.PoolManager(timeout=self.con_timeout, headers=self.http_req_header, ssl_version=ssl.PROTOCOL_TLSv1, ssl_context=ctx) try: if method.lower() == 'get': res = http.request('GET', target_url, fields=query_param, preload_content=preload_content, redirect=self.redirect) else: encoded_args = urlencode(body_param, encoding=enc) res = http.request('POST', target_url + '?' + encoded_args, preload_content=preload_content, redirect=self.redirect) for header in res.headers.items(): res_header += header[0] + ': ' + header[1] + '\r\n' if header[0].lower() == 'server': server_header = header[0] + ': ' + header[1] if header[0].lower() == 'content-type': content_type_value = header[1] # Detect encoding. encoding = self.detect_encoding(res.data, content_type_value) # Get response body. res_body = res.data.decode(encoding) except Exception as e: self.print_message(WARNING, 'Use default charset: {}'.format(self.encoding)) encoding = self.encoding self.print_exception(e, 'Access is failure : {}'.format(target_url)) self.write_log(30, 'Accessing is failure : {}'.format(target_url)) return res, server_header, res_header, res_body, encoding
def set_proxy(ip, port): """init urllib3 proxy connection pool""" global HTTP_CONNECTION_POOL HTTP_CONNECTION_POOL = urllib3.ProxyManager("http://%s:%s" % (ip, port), retries=False) output.print_msg("设置代理成功")
with socket.create_connection((hostname, 443)) as sock: # won't work because of how we are using python to filter # todoruleid:disabled-cert-validation with context.wrap_socket(sock, server_hostname=hostname, cert_reqs = sss.CERT_NONE) as ssock: print(ssock.version()) from urllib3 import PoolManager manager = PoolManager(10) r = manager.request('GET', 'http://google.com/') # ruleid:disabled-cert-validation manager = PoolManager(10, cert_reqs = ssl.CERT_OPTIONAL) # ruleid:disabled-cert-validation proxy = ur3.ProxyManager('http://localhost:3128/', cert_reqs = ssl.CERT_NONE) # ruleid:disabled-cert-validation pool = ur3.connectionpool.HTTPSConnectionPool(cert_reqs=ssl.CERT_OPTIONAL) # ruleid:disabled-cert-validation pool = ur3.connection_from_url('someurl', cert_reqs= ssl.CERT_NONE) # ruleid:disabled-cert-validation pool = ur3.connection_from_url('someurl', cert_reqs='NONE') # OK; invalid pool = ur3.connection_from_url('someurl', cert_reqs='CERT NONE') # ruleid:disabled-cert-validation pool = ur3.connection_from_url('someurl', cert_reqs="NONE")
def create_vuln_yearly_db(self, cve_year, last_modified_date): # Get cve list from NVD. self.utility.write_log( 20, '[In] Create yearly vulnerability database [{}]'.format( self.file_name)) target_url = self.nvd_zip_url.replace('*', cve_year) tmp_file = os.path.join(self.nvd_db_dir, 'temp_' + cve_year + '.zip') # Download zip file (include cve list) and uncompress zip file. target_json_name = '' self.utility.write_log(20, 'Accessing : {}'.format(target_url)) self.utility.print_message( OK, 'Get {} CVE list from {}'.format(cve_year, target_url)) http = None ctx = ssl.create_default_context() ctx.set_ciphers('DEFAULT') # ctx.set_ciphers('DEFAULT@SECLEVEL=1') if self.utility.proxy != '': self.utility.print_message( WARNING, 'Set proxy server: {}'.format(self.utility.proxy)) if self.utility.proxy_user != '': headers = urllib3.make_headers( proxy_basic_auth=self.utility.proxy_user + ':' + self.utility.proxy_pass) http = urllib3.ProxyManager(timeout=self.con_timeout, headers=self.http_req_header, proxy_url=self.utility.proxy, proxy_headers=headers) else: http = urllib3.ProxyManager(timeout=self.con_timeout, headers=self.http_req_header, proxy_url=self.utility.proxy) else: http = urllib3.PoolManager(timeout=self.con_timeout, headers=self.http_req_header, ssl_version=ssl.PROTOCOL_TLS, ssl_context=ctx) try: with http.request('GET', target_url, preload_content=False) as res, open( tmp_file, 'wb') as fout: shutil.copyfileobj(res, fout) except Exception as e: self.utility.print_exception( e, 'Access is failure : {}'.format(target_url)) self.utility.write_log( 30, 'Accessing is failure : {}'.format(target_url)) with zipfile.ZipFile(tmp_file, 'r') as downloaded_zip: target_json_name = downloaded_zip.namelist()[0] downloaded_zip.extractall(self.nvd_db_dir) os.remove(tmp_file) # Create cve list of cve file. yearly_cve_list = [] with codecs.open(os.path.join(self.nvd_db_dir, target_json_name), 'r', encoding='utf-8') as fin: self.extract_vuln_info(json.loads(fin.read().replace('\0', '')), cve_year, last_modified_date) self.utility.write_log( 20, '[Out] Create yearly vulnerability database [{}]'.format( self.file_name)) return yearly_cve_list
def thread_function(test): proxy = urllib3.ProxyManager('http://'+test) r=proxy.request('GET', 'https://www.youtube.com')
#!/usr/bin/env python3 import urllib3 pool = urllib3.PoolManager(10) PROXY_ADDRESS = "http://165.24.10.8:8080" proxy = urllib3.ProxyManager(PROXY_ADDRESS) response = proxy.request('GET', 'http://www.packtpub.com') print(response.status) response.headers.keys() response.headers.values() for header, value in response.headers.items(): print(header + ":" + value)
import urllib3 from bs4 import BeautifulSoup as bs http = urllib3.ProxyManager('https://wwwproxy.sandia.gov:80') base_url = 'https://petsearch.animalhumanenm.org/search/searchResults.asp?s=adoption&searchTypeId=4&animalType=2%2C15&statusID=3&submitbtn=Find+Animals&pagesize=16&task=view&tpage=' first_page_content = bs(http.request('GET', base_url + '1').data)
if results.onion != None: # if search terms set in terminal then change from default to that onionsite = results.onion # set from argparse above in globals section #TOR SETUP GLOBAL Vars SOCKS_PORT = 9050 # TOR proxy port that is default from torrc, change to whatever torrc is configured to # Set socks proxy and wrap the urllib module #socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', SOCKS_PORT) # sets default proxy for connect #socket.socket = socks.socksocket # sets default socket to be the sockipy socket # Perform DNS resolution through the socket #def getaddrinfo(*args): # return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] #socket.getaddrinfo = getaddrinfo header = { 'User-Agent': 'JAMES CAMPBELL jamescampbell.us SEARCH BOT! I FOUND YOU!!!!' } proxy = urllib3.ProxyManager( 'http://127.0.0.1:8119/' ) # using this with privoxy and forwarding to tor, but you could use the other code above and it works fine as well r1 = proxy.request('GET', onionsite, headers=header) print(r1.status) # status code print(r1.headers) # header data print(r1.data.decode('utf8')) # html raw output souper = BeautifulSoup(r1.data, "html.parser") soupera = souper.find_all('a') # get all a href's for eachone in soupera: print('This is a link: \n', eachone.text) exit() # test connect to DuckDuckGo .onion site
def __init__( self, token, index, http_event_server, host="", http_event_port="8088", http_event_server_ssl=True, http_event_collector_ssl_verify=True, max_bytes=MAX_CONTENT_BYTES, proxy=None, timeout=9.05, disk_queue=False, disk_queue_size=MAX_DISKQUEUE_SIZE, disk_queue_compression=5, max_queue_cycles=80, max_bad_request_cycles=40, outage_recheck_time=300, num_fails_indicate_outage=10, ): self.max_queue_cycles = max_queue_cycles self.max_bad_request_cycles = max_bad_request_cycles self.outage_recheck_time = outage_recheck_time self.num_fails_indicate_outage = num_fails_indicate_outage self.retry_diskqueue_interval = 60 self.timeout = timeout self.token = token self.default_index = index self.batch_events = [] self.max_byte_length = max_bytes self.current_byte_length = 0 self.server_uri = [] if proxy: proxy_s = proxy.split("://") if len(proxy_s) == 2: # the form http://host:port is preferred, so we can just use # the value directly although this does work, essentially all # proxy servies will 403 the CONNECT to an https endpoint over # http; so an http://proxy setting may be completely useless # # No, the above is unique to squid3 proxy in default configs -- # probably for security reasons. a tinyproxy setting of # http://iphere:porthere actually does work! \o/ self.proxy = proxy elif http_event_server_ssl: # the old behavior is to default to https proxy if the hec was https # this may not work for the various tinyproxy and etc running out there # but to make those older configs continue to work, we leave this behavior # however, using a form 'http://hostname:port' will disable it self.proxy = "https://{0}".format(proxy) else: self.proxy = "http://{0}".format(proxy) else: self.proxy = None # Set host to specified value or default to localhostname if no value provided if host: self.host = host else: self.host = get_fqdn() Payload.host = self.host # Build and set server_uri for http event collector # Defaults to SSL if flag not passed # Defaults to port 8088 if port not passed servers = http_event_server if not isinstance(servers, list): servers = [servers] for server in servers: if http_event_server_ssl: self.server_uri.append( self.Server(server, http_event_port, proto="https")) else: self.server_uri.append( self.Server(server, http_event_port, proto="http")) # build headers once self.headers = urllib3.make_headers( keep_alive=True, user_agent="hubble-hec/{0}".format(__version__), accept_encoding=True) self.headers.update({ "Content-Type": "application/json", "Authorization": "Splunk {0}".format(self.token) }) # 2019-09-24: lowered retries from 3 (9s + 3*9s = 36s) to 1 (9s + 9s = 18s) # Each new event could potentially take half a minute with 3 retries. # Since Hubble is single threaded, that seems like a horribly long time. # (When retries fail, we potentially queue to disk anyway.) pm_kw = { "timeout": self.timeout, "retries": urllib3.util.retry.Retry( total=1, # total retries; overrides other counts below connect=3, # number of retires on connection errors read=3, # number of retires on read errors status=3, # number of retires on bad status codes redirect=10, # avoid redirect loops by limiting redirects to 10 respect_retry_after_header=True, ), } if http_event_collector_ssl_verify: pm_kw.update({ "cert_reqs": "CERT_REQUIRED", "ca_certs": certifi.where() }) else: pm_kw.update({"cert_reqs": "CERT_NONE"}) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) if self.proxy: log.debug("using ProxyManager(%s)", self.proxy) self.pool_manager = urllib3.ProxyManager(self.proxy, **pm_kw) else: self.pool_manager = urllib3.PoolManager(**pm_kw) if disk_queue: if IS_FIPS_ENABLED: md5 = hashlib.md5(usedforsecurity=False) else: md5 = hashlib.md5() uril = sorted([x.uri for x in self.server_uri]) for url_ in uril: md5.update(encode_something_to_bytes(url_)) actual_disk_queue = os.path.join(disk_queue, md5.hexdigest()) log.debug("disk_queue for %s: %s", uril, actual_disk_queue) self.queue = DiskQueue(actual_disk_queue, size=disk_queue_size, compression=disk_queue_compression) else: self.queue = NoQueue()
import time import re import os import MongoDB from PIL import Image from urllib3.contrib.socks import SOCKSProxyManager url = "https://game.nihaoma.top/t1/?template=blue&token=b595fa57&CateID=10#/" http = urllib3.PoolManager( cert_reqs = 'CERT_REQUIRED', ca_certs = certifi.where() ) ######## 访问某些网站使用本地ssr代理 proxy = urllib3.ProxyManager('http://localhost:1080') sockproxy = SOCKSProxyManager('socks5://localhost:1080') ''' 通过ssr代理访问 端口1080 无Referer ''' def visitByProxy(url): try: web = proxy.request('GET', url, headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36 LBBROWSER'
def make_request(method, host, url, username, password, fields=None, skip_decode=False, get_params={}): # Initialize http and https pool managers num_pools = 1 managers = {} if host.lower().startswith("http://"): scheme = "http" if "http_proxy" in os.environ: proxy_url = urllib3.util.url.parse_url(os.environ["http_proxy"]) managers["http"] = urllib3.ProxyManager( proxy_url=proxy_url.url, proxy_headers=urllib3.util.make_headers( user_agent=user_agent_identifier(), proxy_basic_auth=proxy_url.auth), num_pools=num_pools) else: managers["http"] = urllib3.PoolManager(num_pools=num_pools) elif host.lower().startswith("https://"): scheme = "https" if "https_proxy" in os.environ: proxy_url = urllib3.util.url.parse_url(os.environ["https_proxy"]) managers["https"] = urllib3.ProxyManager( proxy_url=proxy_url.url, proxy_headers=urllib3.util.make_headers( user_agent=user_agent_identifier(), proxy_basic_auth=proxy_url.auth), num_pools=num_pools, cert_reqs=CERT_REQUIRED, ca_certs=certs_file()) else: managers["https"] = urllib3.PoolManager(num_pools=num_pools, cert_reqs=CERT_REQUIRED, ca_certs=certs_file()) else: raise Exception("Unknown scheme") charset = None headers = urllib3.util.make_headers(basic_auth='{0}:{1}'.format( username, password), accept_encoding=True, user_agent=user_agent_identifier(), keep_alive=True) response = None try: manager = managers[scheme] # All arguments must be bytes, not unicode encoded_request = encode_args(manager.request) response = encoded_request(method, host + url, headers=dict(headers), fields=fields) data = response.data if not skip_decode: charset = determine_charset(response) if isinstance(data, bytes): data = data.decode(charset) if response.status < 200 or response.status >= 400: if response.status in (401, 403): raise HttpNotAuthorized(data) elif response.status == 404: raise HttpNotFound(data) else: raise Exception(data) return data, charset except SSLError: logger.error("Invalid SSL certificate") raise finally: if response is not None: response.close()
# async() print('sending chunk to async sync_file') print(len(pickle.dumps(chunk,-1))) print(chunk) # async() print('----') if proxy_url is None: httpClient = None else: import urllib3 httpClient = urllib3.ProxyManager( proxy_url, timeout=urllib3.Timeout.DEFAULT_TIMEOUT, cert_reqs='CERT_REQUIRED', retries=urllib3.Retry( total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504] ) ) client = Minio(endpoint_domain, access_key=aws_access_key, secret_key=aws_secret_key, http_client=httpClient) objcount = 0 folders = set() chunk = {} for obj in client.list_objects_v2(path, prefix=prefix, recursive=True): # print(obj) # add this object's parent to folders map if not (obj.object_name.endswith('/')):
username = sys.argv[4] elif sys.argv[3] == '-U': username = [] for i in open(sys.argv[4], 'r').readlines(): username.append(i.rstrip()) else: print Usage #deal with password args if sys.argv[5] == '-p': password = sys.argv[6] elif sys.argv[5] == '-P': password = [] for i in open(sys.argv[6], 'r').readlines(): password.append(i.rstrip()) else: print Usage stream = urllib3.ProxyManager("http://127.0.0.1:8080/") #stream=urllib3.PoolManager() for i, (u, p) in enumerate(zip(username, password)): req = stream.request(method, target, fields={'username': u, 'password': p}) print req.status if '<td><input type="submit" name="submit" value="Log In" class="button"></td>' in req.data: # put some test string here from response. print '[!] Wrong creds' else: print '[!] found creds--> %s:%s' % (u, p) #print stream.headers ### 0x10w1eve1 ###
try: with open(settings_file, 'r') as f: s = f.read() settings = json.loads(s) if settings['token'] is None: exit() except FileNotFoundError: with open(settings_file, 'w') as f: settings = {'token': None, 'use_proxy': False} json.dump(settings, f, indent=4) exit() if settings['use_proxy']: proxy_url = "http://proxy.server:3128" telepot.api._pools = { 'default': urllib3.ProxyManager(proxy_url=proxy_url, num_pools=3, maxsize=10, retries=False, timeout=30), } telepot.api._onetime_pool_spec = (urllib3.ProxyManager, dict(proxy_url=proxy_url, num_pools=1, maxsize=1, retries=False, timeout=30)) mybot = ChatHandler(Chat, settings['token'], 'db.json') mybot.start()
def __init__(self, configuration, pools_size=4, maxsize=None): # urllib3.PoolManager will pass all kw parameters to connectionpool # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/poolmanager.py#L75 # noqa: E501 # https://github.com/shazow/urllib3/blob/f9409436f83aeb79fbaf090181cd81b784f1b8ce/urllib3/connectionpool.py#L680 # noqa: E501 # maxsize is the number of requests to host that are allowed in parallel # noqa: E501 # Custom SSL certificates and client certificates: http://urllib3.readthedocs.io/en/latest/advanced-usage.html # noqa: E501 # cert_reqs if configuration.verify_ssl: cert_reqs = ssl.CERT_REQUIRED else: cert_reqs = ssl.CERT_NONE # ca_certs if configuration.ssl_ca_cert: ca_certs = configuration.ssl_ca_cert else: # if not set certificate file, use Mozilla's root certificates. ca_certs = certifi.where() addition_pool_args = {} if configuration.assert_hostname is not None: addition_pool_args[ 'assert_hostname'] = configuration.assert_hostname # noqa: E501 if maxsize is None: if configuration.connection_pool_maxsize is not None: maxsize = configuration.connection_pool_maxsize else: maxsize = 4 # https pool manager if configuration.proxy: if configuration.proxy_headers: proxy_headers = configuration.proxy_headers elif (configuration.proxy_user and configuration.proxy_pass): proxy_headers = urllib3.make_headers( proxy_basic_auth= f'{configuration.proxy_user}:{configuration.proxy_pass}' # noqa: E501 ) else: proxy_headers = None self.pool_manager = urllib3.ProxyManager( num_pools=pools_size, maxsize=maxsize, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=configuration.cert_file, key_file=configuration.key_file, proxy_url=configuration.proxy, proxy_headers=proxy_headers, **addition_pool_args) else: self.pool_manager = urllib3.PoolManager( num_pools=pools_size, maxsize=maxsize, cert_reqs=cert_reqs, ca_certs=ca_certs, cert_file=configuration.cert_file, key_file=configuration.key_file, **addition_pool_args)
def _prepare_url_request(host, username, password): """ Return a ProxyManager object (as defined in urllib3 [1]) that can be used to perform authorized requests to a specific host. Authorization header is constructed and set using "username" and "password" parameters. Also set the common HTTP headers that we want to be sent with each request. [1]: http://urllib3.readthedocs.io/en/latest/reference/#urllib3.poolmanager.ProxyManager # noqa """ def get_proxy_url(env_proxy, scheme): parsed_proxy = urllib3.util.url.parse_url(env_proxy) return urllib3.util.url.Url( scheme=scheme, auth=parsed_proxy.auth, host=parsed_proxy.host, port=parsed_proxy.port, path=parsed_proxy.path, query=parsed_proxy.query, fragment=parsed_proxy.fragment ) # Initialize http and https pool managers num_pools = 1 managers = {} if host.lower().startswith("http://"): scheme = "http" if os.environ.get("http_proxy"): proxy_url = get_proxy_url(os.environ["http_proxy"], scheme) managers["http"] = urllib3.ProxyManager( proxy_url=proxy_url.url, proxy_headers=urllib3.util.make_headers( user_agent=user_agent_identifier(), proxy_basic_auth=proxy_url.auth), num_pools=num_pools ) else: managers["http"] = urllib3.PoolManager(num_pools=num_pools) elif host.lower().startswith("https://"): scheme = "https" if os.environ.get("https_proxy"): proxy_url = get_proxy_url(os.environ["https_proxy"], scheme) managers["https"] = urllib3.ProxyManager( proxy_url=proxy_url.url, proxy_headers=urllib3.util.make_headers( user_agent=user_agent_identifier(), proxy_basic_auth=proxy_url.auth), num_pools=num_pools, cert_reqs=CERT_REQUIRED, ca_certs=certs_file() ) else: managers["https"] = urllib3.PoolManager( num_pools=num_pools, cert_reqs=CERT_REQUIRED, ca_certs=certs_file() ) else: raise Exception("Unknown scheme") headers = urllib3.util.make_headers( basic_auth='{0}:{1}'.format(username, password), accept_encoding=True, user_agent=user_agent_identifier(), keep_alive=True ) manager = managers[scheme] return headers, manager
def test_http_proxy(proxy,time): pr = urllib3.ProxyManager('http://'+proxy) pr.request('GET', 'https://www.youtube.com', timeout=time)
UNAUTHORIZED_MESSAGE = 'User is not authorized to access this resource with an explicit deny' DEFAULT_REGION = "us-west-2" ONBOARDING_SOURCE = "checkov" SIGNUP_HEADER = { 'Accept': 'application/json', 'User-Agent': 'Mozilla/5.0 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36', 'Content-Type': 'application/json;charset=UTF-8' } try: http = urllib3.ProxyManager(os.environ['https_proxy']) except KeyError: http = urllib3.PoolManager() class BcPlatformIntegration(object): def __init__(self): self.bc_api_key = read_key() self.s3_client = None self.bucket = None self.credentials = None self.repo_path = None self.repo_id = None self.timestamp = None self.scan_reports = [] self.bc_api_url = os.getenv('BC_API_URL',
index = random.randint(0, num_proxies - 1) value = all_proxies[index] proxy['id'] = value[0] proxy['ip'] = value[1] proxy['port'] = value[2] ok = check_proxy(proxy) return proxy else: return ProxyItem() def data_count(self): sql="select count(*) from proxy" params=() return self.dbHelper.count(sql,*params) def data_select(self): sql="select * from proxy" params=() return self.dbHelper.select(sql,*params) def data_clear(self): params=("proxy",) self.dbHelper.clear(*params) if __name__=='__main__': url = "http://www.baidu.com/js/bdsug.js?v=1.0.3.0" prox_url = "http://%s:%s" % ('185.153.248.130', '80') http = urllib3.ProxyManager(prox_url) r = http.request('GET', url) print(r.status)