def pool_for(self, scheme, host, port): """ Given a URL (from which a scheme and host can be extracted), return a connection pool (potentially with TLS state) which can be used to connect to the URL. """ if scheme is None: if self._keyfile: return urllib3.HTTPSConnectionPool(host, port, key_file=self._keyfile, cert_file=self._certfile, ca_certs=self._cafile) else: return urllib3.HTTPConnectionPool(host, port) elif scheme == "http": return urllib3.HTTPConnectionPool(host, port) elif scheme == "https": if self._keyfile: return urllib3.HTTPSConnectionPool(host, port, key_file=self._keyfile, cert_file=self._certfile, ca_certs=self._cafile) else: raise ValueError("SSL requested without providing certificate") exit(1) elif scheme == "file": # FIXME what to do here? raise ValueError("Unsupported scheme "+scheme) else: raise ValueError("Unsupported scheme "+scheme)
def main(): """Main function.""" # Gather all backlink information from the rest timestamp = datetime.datetime.now().strftime("%y-%m-%d") stats_dir = "/popularity_stats/" + timestamp + "/" url = 'http://127.0.0.1:45454/address/online/' pool = urllib3.HTTPConnectionPool("127.0.0.1", 45454, timeout=10, cert_reqs='CERT_NONE', assert_hostname=False) links = pool.request('GET', url).data links = links.replace(".onion/", "").replace("http://", "").split('\n') for onion_id in links: try: # Random delay 3min + 1-60 seconds delay_time = 180 + random.randrange(1, 60) time.sleep(delay_time) if not onion_id: continue content_type = {'Content-Type': 'application/json'} onion_url = 'http://' + onion_id + '.onion/' print onion_url backlinks = str(get_backlinks(onion_url)) url = 'http://127.0.0.1:45454/address/' + onion_id + "/popularity/" data = '{"date": "' + timestamp + '", "tor2web_access_count": ' data = data + '0, "backlinks": ' + backlinks + '}' print data save_popularity_data(data, onion_id) pool.urlopen('PUT', url, headers=content_type, body=data) except Exception: import traceback print 'generic exception: ' + traceback.format_exc()
def test_request_hook_params(self): def request_hook(span, request, headers, body): span.set_attribute("request_hook_headers", json.dumps(headers)) span.set_attribute("request_hook_body", body) URLLib3Instrumentor().uninstrument() URLLib3Instrumentor().instrument(request_hook=request_hook, ) headers = {"header1": "value1", "header2": "value2"} body = "param1=1¶m2=2" pool = urllib3.HTTPConnectionPool("httpbin.org") response = pool.request("POST", "/status/200", body=body, headers=headers) self.assertEqual(b"Hello!", response.data) span = self.assert_span() self.assertIn("request_hook_headers", span.attributes) self.assertEqual(span.attributes["request_hook_headers"], json.dumps(headers)) self.assertIn("request_hook_body", span.attributes) self.assertEqual(span.attributes["request_hook_body"], body)
def main(): parser = argparse.ArgumentParser() parser.add_argument('host', type=str) parser.add_argument('port', type=int) parser.add_argument('x', type=int) parser.add_argument('y') #, type=int) args = parser.parse_args() http = urllib3.HTTPConnectionPool(args.host, args.port) encoded_args = urlencode({'x': args.x, 'y': args.y}) url = 'http://' + args.host + ':' + str(args.port) + '/?' + encoded_args r = http.request('POST', url) data = r.data.decode('utf-8') hit = 0 sink = '' print(r.status) if r.status == 200: split = data.split('=') if len(split[1]) > 1: hit = int(split[1][0]) sink = split[2] else: hit = int(split[1]) update_opponent_board(args.x, int(args.y), hit) print('Hit: {}'.format(hit)) if sink != '': print('Sink: {}'.format(sink))
def __init__(self, url, maxsize, proxy=None): self.url = url self.maxsize = maxsize self.proxy = proxy if url.find('://') < 0: url = 'http://' + url p = urlparse.urlsplit(url) i = p.netloc.rfind('@') self.host = p.netloc[i + 1:] if i >= 0: self.auth = ('Basic ' + base64.b64encode(p.netloc[:i].encode('U8')).decode()) else: self.auth = None self.path = urlparse.urlunsplit(('', '', p.path or '/', p.query, '')) self.fullpath = 'http://' + self.host + self.path if proxy: self.path = self.fullpath self.pool = urllib3.HTTPConnectionPool(proxy if proxy else self.host, maxsize=maxsize, block=True) self.lp_header = None self.lp_pool = self.pool self.lp_host = self.host self.lp_path = self.path self.expire = 0
def setUp(self): self.http = urllib3.HTTPConnectionPool('127.0.0.1', port=testenv["wsgi_port"], maxsize=20) self.recorder = tracer.recorder self.recorder.clear_spans() tracer._scope_manager = GeventScopeManager()
def __init__(self): esurl = Configure.configure().value("elasticsearch.url") poolsize = Configure.configure().value("elasticsearch.pool.maxsize") self._es_url = esurl self._pool_maxsize = poolsize self._es_domain = None if esurl.startswith("http://"): self._es_domain = esurl[7:] elif esurl.startswith("https://"): self._es_domain = esurl[8:] idx = self._es_domain.find("/") self._es_domain = self._es_domain[:idx] # httpexp = re.compile(r'^https?://') # httpma = httpexp.search(self._es_url) # print (httpma.span()) # s,e=httpma.span() # domain = p_es_url[e:] # i = domain.find("/") # if i >= 0: # domain=domain[:i] # print (domain) print("extract domain", self._es_domain) self._es_client = urllib3.HTTPConnectionPool( self._es_domain, maxsize=self._pool_maxsize)
def __init__(_self, maxsize: int, block: bool, **connection_pool_kwargs): connection_pool_kwargs.get("headers", {}).update({ "connection": "keep-alive", "user-agent": "telegram-bot-client: A Telegram Bot API Python client", }) # from urllib3.connection.default_socket_options from socket import IPPROTO_TCP, SOL_SOCKET, TCP_NODELAY, SO_KEEPALIVE connection_pool_kwargs[ "socket_options"] = connection_pool_kwargs.get( "socket_options", []) + [ (IPPROTO_TCP, TCP_NODELAY, 1), (SOL_SOCKET, SO_KEEPALIVE, 1), ] if self.host.startswith("https://"): _self.pool = urllib3.HTTPSConnectionPool( host=self.host[8:], maxsize=maxsize, block=block, **connection_pool_kwargs) elif self.host.startswith("http://"): _self.pool = urllib3.HTTPConnectionPool( host=self.host[7:], maxsize=maxsize, block=block, **connection_pool_kwargs) else: raise TelegramBotException( "Telegram Bot API only supports https:// and http://")
def __init__(self, cfg): self.simhash_size = cfg['simhash']['size'] self.simhash_expire = cfg['simhash']['expire_after'] if self.simhash_size > 512: raise Exception('do not support simhash longer than 512') headers = { 'User-Agent': 'wayback-discover-diff', 'Accept-Encoding': 'gzip,deflate', 'Connection': 'keep-alive' } cdx_auth_token = cfg.get('cdx_auth_token') if cdx_auth_token: headers['cookie'] = 'cdx_auth_token=%s' % cdx_auth_token self.http = urllib3.HTTPConnectionPool('web.archive.org', maxsize=50, retries=4, headers=headers) self.redis = StrictRedis(connection_pool=BlockingConnectionPool. from_url(cfg['redis_uri'], max_connections=50, timeout=cfg.get('redis_timeout', 10), decode_responses=True)) self.tpool = ThreadPoolExecutor(max_workers=cfg['threads']) self.snapshots_number = cfg['snapshots']['number_per_year'] self.download_errors = 0 # Initialize logger self._log = logging.getLogger('wayback_discover_diff.worker')
def _get_connection_pool(): if DEBUG: return urllib3.HTTPConnectionPool('127.0.0.1:8000') else: return urllib3.HTTPSConnectionPool('api.nivad.io', cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
def test_basic_http_absolute_url(self): url = "http://httpbin.org:666/status/200" httpretty.register_uri(httpretty.GET, url, body="Hello!") pool = urllib3.HTTPConnectionPool("httpbin.org", port=666) response = pool.request("GET", url) self.assert_success_span(response, url)
def test_url_open_explicit_arg_parameters(self): url = "http://httpbin.org:666/status/200" httpretty.register_uri(httpretty.GET, url, body="Hello!") pool = urllib3.HTTPConnectionPool("httpbin.org", port=666) response = pool.urlopen(method="GET", url="/status/200") self.assert_success_span(response, url)
def request(self, method, body, headers, lp=False): headers['host'] = self.lp_host if lp else self.host if self.auth: headers['authorization'] = self.auth elif 'authorization' in headers: del headers['authorization'] pool = self.lp_pool if lp else self.pool r = pool.urlopen(method, self.lp_path if lp else self.path, body, headers, assert_same_host=False, timeout=self.lp_timeout if lp else self.timeout) for h in ('content-encoding', 'transfer-encoding'): if h in r.headers: del r.headers[h] if ('x-long-polling' in r.headers and self.lp_header != r.headers['x-long-polling']): self.lp_header = r.headers['x-long-polling'] p = urlparse.urlsplit(self.lp_header) self.lp_host = p.netloc or self.host if self.proxy: self.lp_path = urlparse.urljoin(self.fullpath, self.lp_header) else: self.lp_path = urlparse.urlunsplit(('', '', p.path or '/', p.query, '')) if self.lp_host == self.host: self.lp_pool = self.pool else: self.lp_pool = urllib3.HTTPConnectionPool( p.netloc, maxsize=self.maxsize, block=True) return r
def play_sessions(runner): runner.http = urllib3.HTTPConnectionPool('localhost:8080', maxsize=2) for i in xrange(0, runner.nsessions): # Select a user runner.user = random.choice(runner.users) # Play a session runner.playSession() return runner.counter
def getInfo(username, password, host): http = urllib3.HTTPConnectionPool(host) # Login request = http.request('GET','/asp/GetRandCount.asp') ## Parameters password64 = base64.b64encode(str.encode(password)).decode('ascii')
def snipplets(): ''' Code for pulling random images from specified boards on 4chan and pushing them to a Apple TV 2 Device ''' #Specify boards boards = ['b','g','v','wg','fa','trv','an','ck'] #Specify Transitions and airplay device IP transitions = ['None', 'SlideLeft', 'SlideRight', 'Dissolve'] airpush_ip = '192.168.1.100' airpush_port = '7000' airpush_target = "/photo" #Uses one connection. Urllib2 closes and reopens connections, this is a bad time. Don't have a bad time. http_pool = urllib3.HTTPConnectionPool(airpush_ip+':'+airpush_port) #Track seen images so we dont get dubs seen = set() #For the purpose of 'social coding' this infinate loop has been replaced to looping 10x #while True: for i in range(10): try: chosen_one = random.choice(boards) #load thread json url = 'http://api.4chan.org/'+chosen_one+'/'+str(random.choice(range(10)))+'.json' urllib2.Request(url) html = urllib2.urlopen(req) data = json.loads(html.read()) #get random thread thread = random.choice(data['threads']) url = 'http://api.4chan.org/'+chosen_one+'/res/'+str(thread['posts'][0]['no'])+'.json' data = getJson(url) #Get random image from thread images = [] for post in data['posts']: if 'tim' in post.keys(): if 'ext' != '.gif': if str(post['tim'])+post['ext'] not in seen: seen.add(str(post['tim'])+post['ext']) images.append(str(post['tim'])+post['ext']) if len(images) > 0: image = random.choice(images) img = urllib2.urlopen("http://images.4chan.org/"+chosen_one+"/src/"+image) #Make request to airplay device r = http_pool.urlopen('PUT',airpush_target,body=img.read(),headers={'X-Apple-Transition': random.choice(transitions)}) #display it on screen for 2secs time.sleep(2) except urllib2.HTTPError: print "404lol" print url
def pattern(self, method, url, params=None, body=None, host='localhost', port=9200, **kwargs): # ... self.pool = urllib3.HTTPConnectionPool(host, port=port) url = self.url_prefix + url if params: url = '%s?%s' % (url, urlencode(params or {})) response = self.pool.urlopen(method, url, body)
def internet_on(): try: http = urllib3.HTTPConnectionPool('www.google.com') http.urlopen(url='http://www.google.com', timeout=1.0, method='GET') logging.info('network is ON') return True except urllib3.exceptions.HTTPError as err: logging.error('network is OFF') return False
def get(self, host, params=()): # type: (object, object) -> object """Get metadata by url""" self.__is_server_online(host) self.__disable_verbose() self.__parse_params(params) scheme, host = urlparse(host).scheme, urlparse(host).netloc self.DEFAULT_HTTP_PROTOCOL = scheme + "://" self.urls = self.__get_urls(host) response = {} self.HEADER['user-agent'] = self.reader.get_random_user_agent() log.info("user-agent : " + self.HEADER['user-agent']) log.info('Thread num : ' + str(self.threads)) try: httplib.HTTPConnection.debuglevel = self.debug if hasattr(urllib3, 'disable_warnings'): urllib3.disable_warnings() if scheme == "http": self.http = urllib3.HTTPConnectionPool( host.split(':')[0], port=80 if len(host.split(':')) == 1 else int( host.split(':')[1]), block=True, maxsize=10) elif scheme == "https": self.http = urllib3.HTTPSConnectionPool( host.split(':')[0], port=443 if len(host.split(':')) == 1 else int( host.split(':')[1]), block=True, maxsize=10) else: log.critical("not support http protocl, Exit now ") sys.exit(1) pool = threadpool.ThreadPool(self.threads) requests = threadpool.makeRequests(self.request, self.urls) for req in requests: pool.putRequest(req) time.sleep(1) pool.wait() except exceptions.AttributeError as e: log.critical(e.message) except KeyboardInterrupt: log.warning('Session canceled') sys.exit() self.counter['total'] = self.urls.__len__() self.counter['pools'] = pool.workers.__len__() response['count'] = self.counter response['result'] = self.result return response
def _create_connection_pool(self): headers = {} headers.setdefault('USER-AGENT', 'SIMBA APP') headers.setdefault('Content-Type', 'application/json') headers.setdefault('Accept-Type', 'application/json') self._http_connection_pool = urllib3.HTTPConnectionPool(host=self.config.get('AUTHENTICATION_HOST'), headers=headers, block=True, maxsize=self.config.get( 'AUTHENTICATION_HOST_MAX_CONNECTIONS'))
def test_basic_http_success_using_connection_pool(self): with urllib3.HTTPConnectionPool(self.http_host, timeout=3) as pool: response = pool.request("GET", "/status/200") self.assert_success_span(response, self.http_url) # Test that when re-using an existing connection, everything still works. # Especially relevant for IP capturing. response = pool.request("GET", "/status/200") self.assert_success_span(response, self.http_url)
def __init__(self, host, timeout=1000, maxsize=3): ''' 初始化 @param host: 服务器地址 @param timeout: 连接超时时间,单位为秒 ''' self.__host = host import urllib3 self.__pool = urllib3.HTTPConnectionPool(host, timeout=timeout, maxsize=maxsize)
def test_urlfetch_called_with_http(self): """Check that URLFetch is used to fetch non-https resources.""" resp = MockResponse('OK', 200, False, 'http://www.google.com', {'content-type': 'text/plain'}) fetch_patch = patch('google.appengine.api.urlfetch.fetch', return_value=resp) with fetch_patch as fetch_mock: import urllib3 pool = urllib3.HTTPConnectionPool('www.google.com', '80') r = pool.request('GET', '/') self.assertEqual(r.status, 200, r.data) self.assertEqual(fetch_mock.call_count, 1)
def do2(): print("urllib3") conn = urllib3.HTTPConnectionPool(domen, maxsize=M) @g_async def f(): for i in range(N): if i % (N // 10) == 0: print(f"{i/N:.0%}") rp: urllib3.HTTPResponse = conn.request("GET", url) trash = rp.data gevent.wait([f() for _ in range(M)])
def __init__(self, host: str, port: int, queue: str, vc_format: str = "base64"): self._host = host self._port = port self._queue_name = queue self._vc_format = vc_format self._url = f"http://{self._host}:{self._port}/queuename/{self._queue_name}?object_format=internal" self._http = urllib3.HTTPConnectionPool(host=self._host, port=self._port, retries=False)
def test_urlfetch_called_with_http(self): """Check that URLFetch is used to fetch non-https resources.""" resp = MockResponse("OK", 200, False, "http://www.google.com", {"content-type": "text/plain"}) fetch_patch = patch("google.appengine.api.urlfetch.fetch", return_value=resp) with fetch_patch as fetch_mock: import urllib3 pool = urllib3.HTTPConnectionPool("www.google.com", "80") r = pool.request("GET", "/") assert r.status == 200, r.data assert fetch_mock.call_count == 1
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--mode", help= "monitoring mode , cluster is cluster health and tps , node is node health and gc infomation (default cluster)", default="cluster", required=False) parser.add_argument("--url", help="the url of cluster (default localhost)", default="localhost", required=False) parser.add_argument("--port", help="the port of cluster (default 9200)", default=9200, required=False) parser.add_argument("--interval", help="the interval of stat api request (default 1s)", default=1, required=False, type=int) args = parser.parse_args() mode = args.mode url = args.url port = args.port interval = args.interval try: esRestSession = urllib3.HTTPConnectionPool(url, maxsize=1, port=port, timeout=10) esRestSession.request('GET', '/') except: printError("ElasticSearch is not installed on " + url + ":" + str(port)) exit(1) if mode == "node": monitoringNodeStatus(esRestSession, interval) else: monitoringClusterStatus(esRestSession, interval)
def __init__(self, config): """ Initialise the Gisgraphy interface. :param config: The :class:`configparser` object. """ url = "localhost" self.post_data = {'q': '', 'format': 'json', 'suggest': 'true'} self.pool = urllib3.HTTPConnectionPool( host=url, port=8080, maxsize=25, headers={'accept': 'application/json'})
def setUp(self): """ Setup sink and confirm replication queue is empty """ self.host = os.getenv('RIAK_HOST', 'localhost') self.sink = ReplSink(host=self.host, port=8098, queue='q1_ttaaefs') self.test_data = b'{"test":"data"}' self.http = urllib3.HTTPConnectionPool(host=self.host, port=8098, retries=False) empty = False while not empty: rec = self.sink.fetch() empty = rec.empty
def __init__(self, host, proto="http", port=80, timeout=15, strict=True, retries=None, redirect=True, assert_same_host=False, assert_hostname=None, assert_fingerprint=None, ssl_version=None): if "://" in host: host = host.split("://")[1].split("/")[0] socket.gethostbyname(host) self.request_cfg = {"timeout": timeout, "retries": retries, "redirect": redirect, "assert_same_host": assert_same_host} if proto.lower() == "https": self.connection = urllib3.HTTPSConnectionPool(host, port=port, timeout=timeout, retries=retries, ssl_version=ssl_version, assert_hostname=assert_hostname, assert_fingerprint=assert_fingerprint) else: self.connection = urllib3.HTTPConnectionPool(host, port=port, timeout=timeout, retries=retries, strict=strict) self.target = host