def __init__(self,url,parameter,HTTPClients,ClientConnectionPool,task=None): if task is not None: self.celeryTask = task self.celeryTaskId = task.request.id else: self.celeryTask = None self.parameter = parameter self.url = url self.numberHTTPClients = HTTPClients self.numberClientConnectionPool = ClientConnectionPool self.http = HTTPClient.from_url(URL(url),concurrency=self.numberClientConnectionPool) self.clientPool = gevent.pool.Pool(self.numberHTTPClients) self.workQueue = JoinableQueue() self.resultList = {} self.workQueueMax = 0 self.workQueueDone = 0 self.countRequests = 0 self.status_codes = {} self.status_codes_count = {} self.meta = {} self.greenletList = {} self.initAdditionalStructures() self.progressMeta = None self.exitFlag = False self.pauseRequests = False
def test_client_ssl(): client = HTTPClient('github.com', ssl=True) assert client.port == 443 response = client.get('/') assert response.status_code == 200 body = response.read() assert len(body)
def test_network_timeout(): with server(network_timeout): http = HTTPClient(*listener, ssl=True, network_timeout=0.1, ssl_options={'ca_certs': CERT}) with pytest.raises(gevent.ssl.SSLError): response = http.get('/') assert response.status_code == 0, 'should have timed out.'
def test_client_simple(): client = HTTPClient('httpbin.org') assert client.port == 80 response = client.get('/') assert response.status_code == 200 body = response.read() assert len(body)
def test_timeout_on_connect(): with timeout_connect_server(): http = HTTPClient(*listener, insecure=True, ssl=True, ssl_options={'ca_certs': CERT}) def run(http, wait_time=100): response = http.get('/') gevent.sleep(wait_time) response.read() gevent.spawn(run, http) gevent.sleep(0) e = None try: http2 = HTTPClient(*listener, insecure=True, ssl=True, connection_timeout=0.1, ssl_options={'ca_certs': CERT}) http2.get('/') except gevent.ssl.SSLError as error: e = error except gevent.socket.timeout as error: e = error except: raise assert e is not None, 'should have raised' if isinstance(e, gevent.ssl.SSLError): assert "operation timed out" in str(e)
def test_client_simple(): client = HTTPClient("www.google.fr") assert client.port == 80 response = client.get("/") assert response.status_code == 200 body = response.read() assert len(body)
def test_ssl_fail_invalid_certificate(): certs = os.path.join( os.path.dirname(os.path.abspath(__file__)), "onecert.pem") client = HTTPClient('www.google.fr', ssl_options={'ca_certs': certs}) assert client.port == 443 with pytest.raises(SSLError): #@UndefinedVariable client.get('/')
def test_client_ssl(): client = HTTPClient("www.google.fr", ssl=True) assert client.port == 443 response = client.get("/") assert response.status_code == 200 body = response.read() assert len(body)
def test_ssl_fail_invalid_certificate(): certs = os.path.join( os.path.dirname(os.path.abspath(__file__)), "oncert.pem") client = HTTPClient('github.com', ssl_options={'ca_certs': certs}) assert client.port == 443 with pytest.raises(SSLError) as e_info: client.get('/') assert e_info.value.reason == 'CERTIFICATE_VERIFY_FAILED'
def __init__(self): self._auth = None self._sj_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=20, network_timeout=15 ) self._pl_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), headers_type=dict, concurrency=1, network_timeout=120 )
def test_internal_server_error(): with server(internal_server_error): client = HTTPClient(*listener) response = client.get("/") assert not response.should_keep_alive() assert response.should_close() body = response.read() assert len(body) == response.content_length
def verify_login(token): sig = md5(APP_KEY + token).hexdigest() url = '%stokenKey=%s&sign=%s' % (KUAIYONG_URL, token, sig) logger.debug('kuaiyong url:%s', url) url = URL(url) http = HTTPClient(url.host, port=url.port) response = eval(http.get(url.request_uri).read()) http.close() return response
def __call__(self, *args, **kwargs): assert self.runtime_config url = self.url_object http = HTTPClient(url.host, headers=self.runtime_config["agent"].get_as_dict()) response = http.get(url.path) code = response.status_code log.debug("%s %s", self.url, code) body = response.read() http.close()
def test_https(): href = u'https://192.168.1.136:8080/stream/webrtc' url = URL(href) cli = HTTPClient(url.host, port=url.port, ssl=True, ssl_options={'ca_certs': 'ssl_certificate.crt'}, connection_timeout=5.0, network_timeout=10.0) response = cli.get(url.request_uri) s = response.read() cli.close() print(s)
def verify_login(session, appid): """ 登陆校验 """ url = "%ssession=%s&appid=%s" % (TBT_URL, session, appid) url = URL(url) http = HTTPClient(url.host, port=url.port) response = eval(http.get(url.request_uri).read()) http.close() return response
def __login(passport): """login """ logger.info('player login passport:%s' % passport) domain = '%s:%s' % (SERVER_MA_URL, SERVERS_MA_WEBPORT) url = '%s/verify?passport=%s' % (domain, passport) url = URL(url) http = HTTPClient(url.host, port=url.port) response = eval(http.get(url.request_uri).read()) http.close() if response.get('result') is True: return str({'result': True, 'account_id': '\'%s\'' % passport}) return str({'result': False})
def test_timeout_during_chuncked_readline(): with server(timeout_during_chuncked_readline): client = HTTPClient(*listener, network_timeout=0.1) response = client.get('/') assert response['transfer-encoding'] == 'chunked' chunks = [] with pytest.raises(gevent.socket.timeout): #@UndefinedVariable data = 'enter_loop' while data: data = response.readline() chunks.append(data) assert len(chunks) == 3
def test_close_during_chuncked_readline(): with server(close_during_chuncked_readline): client = HTTPClient(*listener) response = client.get("/") assert response["transfer-encoding"] == "chunked" chunks = [] with pytest.raises(HTTPException): data = "enter_loop" while data: data = response.readline() chunks.append(data) assert len(chunks) == 3
def test_timeout_during_chuncked_readline(): with server(timeout_during_chuncked_readline): client = HTTPClient(*listener, network_timeout=0.1) response = client.get("/") assert response["transfer-encoding"] == "chunked" chunks = [] with pytest.raises(gevent.socket.timeout): data = "enter_loop" while data: data = response.readline() chunks.append(data) assert len(chunks) == 3
def test_file_post(): body = tempfile.NamedTemporaryFile("a+b", delete=False) name = body.name try: body.write(b"123456789") body.close() with wsgiserver(check_upload(b"123456789", 9)): client = HTTPClient(*listener) with open(name, 'rb') as body: client.post('/', body) finally: os.remove(name)
def test_close_during_chuncked_readline(): with server(close_during_chuncked_readline): client = HTTPClient(*listener) response = client.get('/') assert response['transfer-encoding'] == 'chunked' chunks = [] with pytest.raises(HTTPException): #@UndefinedVariable data = 'enter_loop' while data: data = response.readline() chunks.append(data) assert len(chunks) == 3
def test_readline_multibyte_sep(): with server(readline_multibyte_sep): client = HTTPClient(*listener, block_size=1) response = client.get("/") lines = [] while True: line = response.readline("\r\n") if not line: break data = json.loads(line[:-1]) lines.append(data) assert len(lines) == 100 assert map(lambda x: x["index"], lines) == range(0, 100)
def test_readline_multibyte_sep(): with server(readline_multibyte_sep): client = HTTPClient(*listener, block_size=1) response = client.get('/') lines = [] while True: line = response.readline(b"\r\n") if not line: break data = json.loads(line[:-1].decode()) lines.append(data) assert len(lines) == 100 assert [x['index'] for x in lines] == [x for x in range(0, 100)]
def test_readline_multibyte_splitsep(): with server(readline_multibyte_splitsep): client = HTTPClient(*listener, block_size=1) response = client.get("/") lines = [] last_index = 0 while True: line = response.readline("\r\n") if not line: break data = json.loads(line[:-2]) assert data["a"] == last_index + 1 last_index = data["a"] len(lines) == 3
def __init__(self): self._auth = None self._sj_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), concurrency=20, network_timeout=15, ssl_options={ "ca_certs": certifi.where(), }) self._pl_client = HTTPClient.from_url( "https://{0}{1}".format(SJ_DOMAIN, SJ_URL), concurrency=1, network_timeout=120, ssl_options={ "ca_certs": certifi.where(), })
def rpc(method, params=None, abort_on_error=False): endpoint = config.BLOCKCHAIN_SERVICE_CONNECT auth = None m = re.search('(.*?//)(.*?):(.*?)@(.*)', endpoint) if m: endpoint = m.group(1) + m.group(4) auth = (m.group(2), m.group(3)) if not params: params = [] payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } headers = { 'Content-Type': 'application/json', 'Connection':'close', #no keepalive } if auth: #auth should be a (username, password) tuple, if specified headers['Authorization'] = util.http_basic_auth_str(auth[0], auth[1]) try: u = URL(endpoint) client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT, network_timeout=JSONRPC_API_REQUEST_TIMEOUT) r = client.post(u.request_uri, body=json.dumps(payload), headers=headers) except Exception, e: raise Exception("Got call_jsonrpc_api request error: %s" % e)
def call_jsonrpc_api(method, params=None, endpoint=None, auth=None, abort_on_error=False): if not endpoint: endpoint = config.COUNTERPARTYD_RPC if not auth: auth = config.COUNTERPARTYD_AUTH if not params: params = {} payload = { "id": 0, "jsonrpc": "2.0", "method": method, "params": params, } headers = { 'Content-Type': 'application/json', 'Connection':'close', #no keepalive } if auth: #auth should be a (username, password) tuple, if specified headers['Authorization'] = http_basic_auth_str(auth[0], auth[1]) try: u = URL(endpoint) client = HTTPClient.from_url(u, connection_timeout=JSONRPC_API_REQUEST_TIMEOUT, network_timeout=JSONRPC_API_REQUEST_TIMEOUT) r = client.post(u.request_uri, body=json.dumps(payload), headers=headers) except Exception, e: raise Exception("Got call_jsonrpc_api request error: %s" % e)
def fire_custom_event(api_key, api_secret, api_method, environment_id, api_call_data): timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") api_key = api_key.encode("utf-8") api_secret = api_secret.encode("utf-8") params = { "Action": api_method, "EnvID": environment_id, "Version": API_VERSION, "AuthVersion": API_AUTH_VERSION, "Timestamp": timestamp, "KeyID": api_key, "Signature": base64.b64encode(hmac.new(api_secret, ":".join([api_method, api_key, timestamp]), hashlib.sha256).digest()), } params.update(api_call_data) url = URL(API_URL) for k, v in params.items(): url[k] = v http = HTTPClient.from_url(url) response = http.get("/?" + url.query_string) code = response.status_code body = response.read() http.close() return url, code, body
def test_response_context_manager(): client = HTTPClient.from_url('http://www.google.fr/') r = None with client.get('/') as response: assert response.status_code == 200 r = response assert r._sock is None # released
def make_client(self, url): o = self.options headers = {'Connection': 'Keep-Alive' if o.keepalive else 'Close'} resolver = Resolver(configure=False) if o.dns: resolver.nameservers = [o.dns] u = urlparse.urlparse(url) qname = u.hostname answer = resolver.query(qname, rdtype=dns.rdatatype.A, rdclass=dns.rdataclass.IN, tcp=False, source=None, raise_on_no_answer=False) if answer.response.answer: ip = answer.response.answer[0].items[0].address if u.port: netloc = '%s:%d' % (ip, u.netloc.split(':')[1]) else: netloc = ip url = urlparse.urlunsplit((u[0], netloc, u[2], u[3], u[4])) headers['Host'] = qname client = HTTPClient.from_url(url, concurrency=o.concurrency, connection_timeout=o.timeout, network_timeout=o.timeout, headers=headers, ssl_options=dict(ssl_version=PROTOCOL_TLSv1, cert_reqs=CERT_NONE) ) return client
def make_stream_request(url): try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) except Exception, e: data = (False, "Got exception: %s" % e)
class UserTsak(TaskSet): def on_start(self): '''初始化数据''' url = URL(host) # 若为https请求,ssl设置为True self.http = HTTPClient(url.host,url.port,ssl=False,connection_timeout=20,network_timeout=20) @task def test(self): try: start_time = time.time() # get 请求 res = self.http.get("/sz/api2/test") # post 请求示例 # body = json.dumps({"username":"******","password":"******"}) # res = self.http.post("/sz/api2/login",body = body) data = res.read() end_time = time.time() response_time =int((end_time - start_time)*1000) response_length = len(data) assert json.loads(data)['Error'] == 0 if res.status_code == 200: # 统计正常的请求 events.request_success.fire(request_type="GET", name="test_success", response_time = response_time, response_length=response_length) # 统计断言失败的请求 except AssertionError: end_time = time.time() response_time =int((end_time - start_time)*1000) events.request_failure.fire(request_type="GET", name="test_failure", response_time=response_time,response_length=0, exception="断言错误。status_code:{}。接口返回:{}。".format(res.status_code,json.loads(data))) # 统计超时异常 except socket.timeout: events.locust_error.fire(locust_instance=UserTsak, exception='Timeout', tb =sys.exc_info()[2]) # 统计其他异常 except Exception as e: events.locust_error.fire(locust_instance=UserTsak, exception='Error:{}。\nstatus_code:{}。\n接口返回:{}。'.format(e,res.status_code,data), tb=sys.exc_info()[2]) def on_stop(self): '''运行结束,关闭http/https连接''' self.http.close()
def make_request(method, params=None): global nonce nonce += 1 # NOQA payload = { "id": nonce, "jsonrpc": "2.0", "method": method, "params": params or [], } payload_data = json.dumps(force_obj_to_text(payload, True)) if 'GETH_ASYNC_GEVENT' in os.environ: from geventhttpclient import HTTPClient client = HTTPClient( host='127.0.0.1', port=open_port, connection_timeout=10, network_timeout=10, headers={ 'Content-Type': 'application/json', }, ) with contextlib.closing(client): response = client.post('/', body=payload_data) response_body = response.read() result = json.loads(force_text(response_body)) else: import requests response = requests.post( endpoint, data=payload_data, headers={ 'Content-Type': 'application/json', }, ) result = response.json() if 'error' in result: raise AssertionError(result['error']) return result['result']
def execute_gevent_requests(number_of_requests: int) -> None: url = URL(SITE) http = HTTPClient.from_url(url, concurrency=1) pool = gevent.pool.Pool(20) for _ in range(1, number_of_requests): pool.spawn(gevent_fetch, http) pool.join() http.close()
def request(method, url, level=0): url = URL(url) http = HTTPClient.from_url(url, concurrency=10) response = getattr(http, method)(url.request_uri) if response.status_code in (301, 302) and level < 3: [loc] = [v for n, v in response.headers if n.lower() == 'location'] return request(method, loc, level+1) result = response.status_code, response.read() http.close() return result
def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5): headers = { 'Connection':'close', } #no keepalive try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers=headers) except Exception, e: raise Exception("Got get_url request error: %s" % e)
def crawler(): http = HTTPClient.from_url(URL(initial_url)) while True: try: num, url = task_list.pop() except IndexError: return else: req = URL(url) res = http.get(req.request_uri) process_response(res.read())
def make_request(self, method, params): from web3 import __version__ as web3_version request_data = self.encode_rpc_request(method, params) request_user_agent = 'Web3.py/{version}/{class_name}'.format( version=web3_version, class_name=type(self), ) client = HTTPClient( host=self.host, port=self.port, ssl=self.ssl, headers={ 'Content-Type': 'application/json', 'User-Agent': request_user_agent, }, ) with contextlib.closing(client): response = client.post(self.path, body=request_data) response_body = response.read() return response_body
def _get_client(host, port, **kwargs): ordered_kwargs = collections.OrderedDict(sorted(kwargs.items())) cache_key = '{0}:{1}:{2}'.format( host, port, ':'.join(( "{0}={1}".format(str(key), str(value)) for key, value in ordered_kwargs.items() )) ) if cache_key not in _client_cache: _client_cache[cache_key] = HTTPClient(host, port, **kwargs) return _client_cache[cache_key]
def test_beehive(self, server, port): """Test redis instance :param server: host name :param port: server port [default=6379] """ url = URL(u'http://%s:%s/v1.0/server/ping/' % (server, port)) http = HTTPClient(url.host, port=url.port) # issue a get request response = http.get(url.request_uri) # read status_code response.status_code # read response body res = json.loads(response.read()) # close connections http.close() if res[u'status'] == u'ok': resp = True else: resp = False self.logger.info(u'Ping beehive %s : %s' % (url.request_uri, resp)) self.json = u'Ping beehive %s : %s' % (url.request_uri, resp)
def get(url): log.debug('proxy: %s', url) opts = dict( insecure=True, ssl_options={'cert_reqs': ssl.CERT_NONE} ) try: http = HTTPClient.from_url(url, **opts) res = http.get(url) except Exception as e: log.error(e) abort(500, repr(e)) return res
def delete_request(self, user_id, request_id): """Deletes the Request with the given ID for the given user.""" path = URL('/%s_%s' % (request_id, user_id)) path['access_token'] = self.access_token http = HTTPClient.from_url(FACEBOOK_URL, connection_timeout=self.timeout) resp = http.delete(path.request_uri) content = resp.read() content = _parse_json(content) if content and isinstance(content, dict) and content.get("error"): raise GraphAPIError(content["error"]) http.close() return content
def __init__(self, app_type: ApplicationType = ApplicationType.IOS, concurrency: int = 30, secondary: bool = False): self.header_builder = HeaderBuilder(app_type, secondary=secondary) self.__concurrency = concurrency url = URL(LEGY_HOST) self.__client = HTTPClient(url.host, url.port, concurrency=self.__concurrency, ssl=True, connection_timeout=DEFAULT_TIMEOUT, network_timeout=DEFAULT_TIMEOUT)
def push_hawkular_metrics(): collector = HawkularCollector(Collector(conn)) http = HTTPClient(args.hawkular_host, args.hawkular_port) while True: try: hawkular_metrics = collections.defaultdict(list) for metrics in collector.collect(): hawkular_metrics[metrics[0]].append(metrics[1]) for metrics_type, metrics in hawkular_metrics: response = http.post('/hawkular/metrics/' + metrics_type + '/raw', json.dumps(metrics), headers={ "Content-Type": "application/json", "Hawkular-Tenant": args.hawkular_tenant }) logging.getLogger('vadvisor').debug(response) except Exception as e: logging.getLogger('vadvisor').error(e) sleep(args.hawkular_interval)
def __init__(self, url, connection_count=1, connection_timeout=60.0, network_timeout=60.0, verbose=False): self._last_request_id = None self._parsed_url = URL("http://" + url) self._client_stub = HTTPClient.from_url( self._parsed_url, concurrency=connection_count, connection_timeout=connection_timeout, network_timeout=network_timeout) self.verbose = verbose
def __init__(self, url: str, headers: Dict[str, str], concurrency: int = 30, client: Optional[HTTPClient] = None): self.__url = URL(url) self.__custom_headers = headers self.__concurrency = concurrency if not client: self.client = HTTPClient(self.__url.host, self.__url.port, concurrency=self.__concurrency, ssl=True) else: self.client = client if self.__url.path != LONG_POLLING_V4_PATH: self.flush = self.__flush else: self.open() self.flush = self.__flush_and_reconnect self.__wbuf = BytesIO()
class CartoDBAPIKey(CartoDBBase): """ this class provides you access to auth CartoDB API using your API. You can find your API key in: https://USERNAME.cartodb.com/your_apps/api_key. this method is easier than use the oauth authentification but if less secure, it is recommended to use only using the https endpoint """ def __init__( self, api_key, cartodb_domain, host='cartodb.com', protocol='https', api_version='v2', proxy_info=None, *args, **kwargs): CartoDBBase.__init__(self, cartodb_domain, host, protocol, api_version) self.api_key = api_key self.client = HTTPClient( '.'.join([cartodb_domain, host]), connection_timeout=10.0, network_timeout=10.0, ssl=protocol == 'https', **kwargs) if protocol != 'https': warnings.warn("you are using API key auth method with http") def req(self, url, http_method="GET", http_headers={}, body=None): if http_method.upper() == "POST": body = body or {} body.setdefault('api_key', self.api_key) headers = {'Content-type': 'application/x-www-form-urlencoded'} headers.update(http_headers) resp = self.client.post( url.request_uri, body=urlencode(body), headers=headers) else: url['api_key'] = self.api_key resp = self.client.get(url.request_uri, headers=http_headers) return resp
def check_recaptcha(secret, resp, ip): try: url = URL( 'https://www.google.com/recaptcha/api/siteverify?secret=%s&response=%s&ip=%s' % (secret, resp, ip)) http = HTTPClient.from_url(url) response = http.get(url.request_uri) if response.status_code == 200: raw_res = response.read() res = json.loads(raw_res) if res.get('success'): return True except: pass return False
def test_timeout_on_connect(): with timeout_connect_server() as listener: http = HTTPClient(*listener, insecure=True, ssl=True, ssl_options={'ca_certs': CERT}) def run(http, wait_time=100): try: response = http.get('/') gevent.sleep(wait_time) response.read() except Exception: pass gevent.spawn(run, http) gevent.sleep(0) e = None try: http2 = HTTPClient(*listener, insecure=True, ssl=True, connection_timeout=0.1, ssl_options={'ca_certs': CERT}) http2.get('/') except gevent.ssl.SSLError as error: e = error except gevent.socket.timeout as error: e = error except: raise assert e is not None, 'should have raised' if isinstance(e, gevent.ssl.SSLError): assert "operation timed out" in str(e)
def __init__(self, url, connection_count=1, connection_timeout=60.0, network_timeout=60.0, verbose=False, max_greenlets=None): self._parsed_url = URL("http://" + url) self._client_stub = HTTPClient.from_url( self._parsed_url, concurrency=connection_count, connection_timeout=connection_timeout, network_timeout=network_timeout) self._pool = gevent.pool.Pool(max_greenlets) self._verbose = verbose
def get_cdkey_gift_1123(data, player): request = cdkey_pb2.CdkeyRequest() request.ParseFromString(data) response = cdkey_pb2.CdkeyResqonse() url = '%s/verify?area_id=%s&uid=%s&code=%s&token=%s' % \ (CDKEY_URL, SERVER_NO, player.base_info.id, request.cdkey, SERVER_TOKEN) logger.debug('cdkey url:%s', url) url = URL(url) http = HTTPClient(url.host, port=url.port) url_response = eval(http.get(url.request_uri).read()) http.close() logger.debug('cdkey url result:%s', url_response) # gain_data = tomorrow_gift['reward'] # return_data = gain(player, gain_data, const.TOMORROW_GIFT) # get_return(player, return_data, response.gain) response.res.result = True response.res.result_no = url_response.get('success') response.res.message = str(url_response.get('message')) return response.SerializeToString()
def proxy(): url = request.query.get('url') if not url: return abort(400) if url.startswith('//'): url = 'https:' + url log.debug('proxy: %s', url) http = HTTPClient.from_url(url) res = http.get(url) response.status = res.status_code for key, val in res.headers: if key in ('content-type', 'content-length'): response.set_header(key, val) return bytes(res.read())
def request(url, method='GET', payload=u'', headers={}, timeout=30): """A super-simplistic http/s client with no SSL cert validation. >>> print request('http://google.com').status_code 301 >>> print request('https://google.com').read() <HTML><HEAD>... """ client = HTTPClient.from_url( url, ssl_options={'cert_reqs': gevent.ssl.CERT_NONE}, # @UndefinedVariable network_timeout=timeout, connection_timeout=timeout) try: return client.request(method, url, payload, headers) finally: client.close()
def get_token(self): url = URL(self.auth_url) headers = {} headers['x-auth-user'] = self.auth_user headers['x-auth-key'] = self.auth_key self.http = HTTPClient.from_url(url, headers=headers) response = self.http.get(url.request_uri) assert response.status_code == 200 self.token = response['x-auth-token'] self.storage_url = response['x-storage-url'] #print 'TOKEN: %s' % self.token #print 'URL: %s' % self.storage_url self.http.close() self.http = None
def swift_auth_v2(self): self.tenant, self.user = self.user.split(";") auth_dict = {} auth_dict["auth"] = { "passwordCredentials": { "username": self.user, "password": self.password, }, "tenantName": self.tenant, } auth_json = json.dumps(auth_dict) headers = {"Content-Type": "application/json"} auth_httpclient = HTTPClient.from_url( self.auth_url, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, ) path = urlparse.urlparse(self.auth_url).path if not path.endswith("tokens"): path = posixpath.join(path, "tokens") ret = auth_httpclient.request("POST", path, body=auth_json, headers=headers) if ret.status_code < 200 or ret.status_code >= 300: raise SwiftException( "AUTH v2.0 request failed on " + "%s with error code %s (%s)" % ( str(auth_httpclient.get_base_url()) + path, ret.status_code, str(ret.items()), )) auth_ret_json = json.loads(ret.read()) token = auth_ret_json["access"]["token"]["id"] catalogs = auth_ret_json["access"]["serviceCatalog"] object_store = [ o_store for o_store in catalogs if o_store["type"] == "object-store" ][0] endpoints = object_store["endpoints"] endpoint = [ endp for endp in endpoints if endp["region"] == self.region_name ][0] return endpoint[self.endpoint_type], token
def test_multi_queries_greenlet_safe(): client = HTTPClient('www.google.fr', concurrency=3) group = gevent.pool.Group() event = gevent.event.Event() def run(i): event.wait() response = client.get('/') return response, response.read() count = 0 gevent.spawn_later(0.2, event.set) for response, content in group.imap_unordered(run, xrange(5)): assert response.status_code == 200 assert len(content) count += 1 assert count == 5
def __init__(self): ''' @description: - initializes DB connections and other configurations ''' self.pool = Pool(1000) # used for spawning threads values = Configuration.values() # -- initializes DB -- self.dbconn = SQLUtils( host=values['mysql-db']['sms_api_config']['host'], port=values['mysql-db']['sms_api_config']['port'], database=values['mysql-db']['sms_api_config']['db'], user=values['mysql-db']['sms_api_config']['user'], password=values['mysql-db']['sms_api_config']['password']) print '-------' print 'DB:' print 'port: %s' % values['mysql-db']['sms_api_config']['host'] print 'host: %s' % values['mysql-db']['sms_api_config']['port'] print 'db: %s' % values['mysql-db']['sms_api_config']['db'] print 'user: %s' % values['mysql-db']['sms_api_config']['user'] print 'password: %s' % values['mysql-db']['sms_api_config']['password'] print '-------' # -- initializes dragonpay related config -- self.api_url = values['dragonpay']['api_url'] self.merchant_id = values['dragonpay']['merchant_id'] self.secret_key = values['dragonpay']['secret_key'] self.host = values['dragonpay']['host'] self.api_get_txn_status_url = values['dragonpay'][ 'api_get_txn_status_url'] self.uri = values['dragonpay']['uri'] parse_command_line() if options.config == 'prod': url = URL('https://%s' % values['dragonpay']['host']) else: url = URL('http://%s' % values['dragonpay']['host']) self.http_conn = HTTPClient.from_url(url, concurrency=10)
def __init__(self, root, conf): """Initialize a SwiftConnector Args: root: The swift container that will act as Git bare repository conf: A ConfigParser Object """ self.conf = conf self.auth_ver = self.conf.get("swift", "auth_ver") if self.auth_ver not in ["1", "2"]: raise NotImplementedError( "Wrong authentication version use either 1 or 2") self.auth_url = self.conf.get("swift", "auth_url") self.user = self.conf.get("swift", "username") self.password = self.conf.get("swift", "password") self.concurrency = self.conf.getint("swift", "concurrency") or 10 self.http_timeout = self.conf.getint("swift", "http_timeout") or 20 self.http_pool_length = self.conf.getint("swift", "http_pool_length") or 10 self.region_name = self.conf.get("swift", "region_name") or "RegionOne" self.endpoint_type = self.conf.get("swift", "endpoint_type") or "internalURL" self.cache_length = self.conf.getint("swift", "cache_length") or 20 self.chunk_length = self.conf.getint("swift", "chunk_length") or 12228 self.root = root block_size = 1024 * 12 # 12KB if self.auth_ver == "1": self.storage_url, self.token = self.swift_auth_v1() else: self.storage_url, self.token = self.swift_auth_v2() token_header = {"X-Auth-Token": str(self.token)} self.httpclient = HTTPClient.from_url( str(self.storage_url), concurrency=self.http_pool_length, block_size=block_size, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, headers=token_header, ) self.base_path = str( posixpath.join( urlparse.urlparse(self.storage_url).path, self.root))
def fetch_image(url, folder, filename, max_size=20*1024, formats=['png'], dimensions=(48, 48), fetch_timeout=1): def make_data_dir(subfolder): path = os.path.join(config.DATA_DIR, subfolder) if not os.path.exists(path): os.makedirs(path) return path try: #fetch the image data try: u = URL(url) client_kwargs = {'connection_timeout': fetch_timeout, 'network_timeout': fetch_timeout, 'insecure': True} if u.scheme == "https": client_kwargs['ssl_options'] = {'cert_reqs': gevent.ssl.CERT_NONE} client = HTTPClient.from_url(u, **client_kwargs) r = client.get(u.request_uri, headers={'Connection':'close'}) raw_image_data = r.read(max_size) #read up to max_size except Exception, e: raise Exception("Got fetch_image request error: %s" % e) else:
def swift_auth_v2(self): self.tenant, self.user = self.user.split(';') auth_dict = {} auth_dict['auth'] = { 'passwordCredentials': { 'username': self.user, 'password': self.password, }, 'tenantName': self.tenant } auth_json = json_dumps(auth_dict) headers = {'Content-Type': 'application/json'} auth_httpclient = HTTPClient.from_url( self.auth_url, connection_timeout=self.http_timeout, network_timeout=self.http_timeout, ) path = urlparse.urlparse(self.auth_url).path if not path.endswith('tokens'): path = posixpath.join(path, 'tokens') ret = auth_httpclient.request('POST', path, body=auth_json, headers=headers) if ret.status_code < 200 or ret.status_code >= 300: raise SwiftException('AUTH v2.0 request failed on ' + '%s with error code %s (%s)' % (str(auth_httpclient.get_base_url()) + path, ret.status_code, str(ret.items()))) auth_ret_json = json_loads(ret.read()) token = auth_ret_json['access']['token']['id'] catalogs = auth_ret_json['access']['serviceCatalog'] object_store = [ o_store for o_store in catalogs if o_store['type'] == 'object-store' ][0] endpoints = object_store['endpoints'] endpoint = [ endp for endp in endpoints if endp["region"] == self.region_name ][0] return endpoint[self.endpoint_type], token