def test_nonstr_header_values(self): origHTTPSConnection = bufferedhttp.HTTPSConnection bufferedhttp.HTTPSConnection = MockHTTPSConnection try: bufferedhttp.http_connect('127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={ 'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': { 'crazy': 'value' } }, ssl=True) bufferedhttp.http_connect_raw('127.0.0.1', 8080, 'GET', '/', headers={ 'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': { 'crazy': 'value' } }, ssl=True) finally: bufferedhttp.HTTPSConnection = origHTTPSConnection
def test_nonstr_header_values(self): with mock.patch('swift.common.bufferedhttp.HTTPSConnection', MockHTTPSConnection): bufferedhttp.http_connect('127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={ 'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': { 'crazy': 'value' } }, ssl=True) bufferedhttp.http_connect_raw('127.0.0.1', 8080, 'GET', '/', headers={ 'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': { 'crazy': 'value' } }, ssl=True)
def test_nonstr_header_values(self): class MockHTTPSConnection(object): def __init__(self, hostport): pass def putrequest(self, method, path, skip_host=0): pass def putheader(self, header, *values): # Essentially what Python 2.7 does that caused us problems. '\r\n\t'.join(values) def endheaders(self): pass origHTTPSConnection = bufferedhttp.HTTPSConnection bufferedhttp.HTTPSConnection = MockHTTPSConnection try: bufferedhttp.http_connect('127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) bufferedhttp.http_connect_raw('127.0.0.1', 8080, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) finally: bufferedhttp.HTTPSConnection = origHTTPSConnection
def test_nonstr_header_values(self): with mock.patch('swift.common.bufferedhttp.HTTPSConnection', MockHTTPSConnection): bufferedhttp.http_connect( '127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) bufferedhttp.http_connect_raw( '127.0.0.1', 8080, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True)
def test_nonstr_header_values(self): origHTTPSConnection = bufferedhttp.HTTPSConnection bufferedhttp.HTTPSConnection = MockHTTPSConnection try: bufferedhttp.http_connect( '127.0.0.1', 8080, 'sda', 1, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) bufferedhttp.http_connect_raw( '127.0.0.1', 8080, 'GET', '/', headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0, 'x-four': {'crazy': 'value'}}, ssl=True) finally: bufferedhttp.HTTPSConnection = origHTTPSConnection
def GET(self, env, start_response): """ Handle GET Data Object request """ # Create a new WebOb Request object according to the current request req = Request(env) ssl = True if req.scheme.lower() == 'https' else False conn = http_connect_raw(req.server_name, req.server_port, 'GET', '/auth/v1.0', req.headers, None, ssl) res = conn.getresponse() # Create a new response resp = Response() if res.status == 200: storage_url = res.getheader('X-Storage-Url') resp.headers['X-Auth-Token'] = res.getheader('X-Auth-Token') resp.headers['X-Storage-Token'] = res.getheader('X-Storage-Token') subs = storage_url.partition('/v1/') resp.headers['X-Storage-Url'] = (req.host_url + '/' + self.cdmi_root + '/' + subs[2]) return resp else: resp.status = res.status return resp
def _connect_put_node(self, host, port, method, path, headers, query_string, ssl=False): try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, method, path, headers=headers, query_string=query_string, ssl=ssl) if headers.has_key('content-length') and int( headers['content-length']) == 0: return conn with Timeout(self.node_timeout): resp = conn.getexpect() if resp.status == 100: return conn elif resp.status == 507: self.logger.error('507 Insufficient Storage in %s:%s%s' % (host, port, path)) raise Exception except: self.logger.error('Expect: 100-continue on %s:%s%s' % (host, port, path)) return None
def GET(self, env, start_response): """ Handle GET Data Object request """ # Create a new WebOb Request object according to the current request req = Request(env) ssl = True if req.scheme.lower() == 'https' else False conn = http_connect_raw(req.server_name, req.server_port, 'GET', '/auth/v1.0', req.headers, None, ssl) res = conn.getresponse() # Create a new response resp = Response() if res.status == 200: storage_url = res.getheader('X-Storage-Url') resp.headers['X-Auth-Token'] = res.getheader('X-Auth-Token') resp.headers['X-Storage-Token'] = res.getheader('X-Storage-Token') subs = storage_url.partition('/v1/') resp.headers['X-Storage-Url'] = ( req.host_url + '/' + self.cdmi_root + '/' + subs[2]) return resp else: resp.status = res.status return resp
def direct_get_recon(node, recon_command, conn_timeout=5, response_timeout=15, headers=None): """ Get recon json directly from the storage server. :param node: node dictionary from the ring :param recon_command: recon string (post /recon/) :param conn_timeout: timeout in seconds for establishing the connection :param response_timeout: timeout in seconds for getting the response :param headers: dict to be passed into HTTPConnection headers :returns: deserialized json response :raises DirectClientReconException: HTTP GET request failed """ if headers is None: headers = {} ip, port = get_ip_port(node, headers) path = '/recon/%s' % recon_command with Timeout(conn_timeout): conn = http_connect_raw(ip, port, 'GET', path, headers=gen_headers(headers)) with Timeout(response_timeout): resp = conn.getresponse() if not is_success(resp.status): raise DirectClientReconException('GET', node, path, resp) return json.loads(resp.read())
def __call__(self): """ :return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure """ if self.headers.has_key('content-length'): if int(self.headers['content-length']) >= MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) parsed = urlparse(self.url) if self.proxy: proxy_parsed = urlparse(self.proxy) if self._proxy_request_check(parsed.path): host, port = self.split_netloc(proxy_parsed) path = self.url else: host, port = self.split_netloc(parsed) path = parsed.path self.headers['host'] = '%s:%s' % (host, port) if self.method == 'PUT' and len(parsed.path.split('/')) == 5: chunked = self.req.headers.get('transfer-encoding') reader = self.req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.chunk_size), '') bytes_transferred = 0 # pile = GreenPile() # pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query) # conns = [conn for conn in pile if conn] # conn = conns[0] try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, self.method, path, headers=self.headers, query_string=parsed.query) with ContextPool(1) as pool: conn.failed = False conn.queue = Queue(10) pool.spawn(self._send_file, conn, path) while True: with ChunkReadTimeout(self.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: conn.queue.put('0\r\n\r\n') break except TypeError, err: self.logger.info('Chunk Read Error: %s' % err) break except Exception, err: self.logger.info('Chunk Read Error: %s' % err) return HTTPServerError(request=self.req) bytes_transferred += len(chunk) if bytes_transferred > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=self.req) if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk)
def check_resource(env, method, path, logger, get_body=False, query_string=None): """ Use this method to check if a resource already exist. If the resource exists, then it should return True with headers. If the resource does not exist, then it should return False with None headers If the get_body is set to True, the response body will also be returned """ # Create a new Request req = Request(env) ssl = True if req.scheme.lower() == 'https' else False # Fixup the auth token, for some reason, the auth token padded the user # account at the front with a comma. We need to get rid of it, otherwise, # the auth token will be considered invalid. headers = {} if req.headers.get(Consts.AUTH_TOKEN): key, sep, value = req.headers[Consts.AUTH_TOKEN].partition(',') headers[Consts.AUTH_TOKEN] = value if value != '' else key headers['Accept'] = 'application/json' method = 'GET' if not method else method path = req.path if not path else path path = path.rstrip('/') conn = http_connect_raw(req.server_name, req.server_port, method, path, headers, query_string, ssl) res = conn.getresponse() if res.status == 404: conn.close() return False, {}, None elif res.status == 200 or res.status == 204: values = {} header_list = res.getheaders() for header in header_list: values[header[0]] = header[1] if get_body: length = res.getheader('content-length') if length: body = res.read(int(length)) else: body = res.read() else: body = "" conn.close() return True, values, body else: values = {} header_list = res.getheaders() for header in header_list: values[header[0]] = header[1] conn.close() return True, values, None
def test_nonstr_header_values(self): class MockHTTPSConnection(object): def __init__(self, hostport): pass def putrequest(self, method, path, skip_host=0): pass def putheader(self, header, *values): # Essentially what Python 2.7 does that caused us problems. "\r\n\t".join(values) def endheaders(self): pass origHTTPSConnection = bufferedhttp.HTTPSConnection bufferedhttp.HTTPSConnection = MockHTTPSConnection try: bufferedhttp.http_connect( "127.0.0.1", 8080, "sda", 1, "GET", "/", headers={"x-one": "1", "x-two": 2, "x-three": 3.0, "x-four": {"crazy": "value"}}, ssl=True, ) bufferedhttp.http_connect_raw( "127.0.0.1", 8080, "GET", "/", headers={"x-one": "1", "x-two": 2, "x-three": 3.0, "x-four": {"crazy": "value"}}, ssl=True, ) finally: bufferedhttp.HTTPSConnection = origHTTPSConnection
def _connect_put_node(self, host, port, method, path, headers, query_string, ssl=False): try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, method, path, headers=headers, query_string=query_string, ssl=ssl) if headers.has_key('content-length') and int(headers['content-length']) == 0: return conn with Timeout(self.node_timeout): resp = conn.getexpect() if resp.status == 100: return conn elif resp.status == 507: self.logger.error('507 Insufficient Storage in %s:%s%s' % (host, port, path)) raise Exception except: self.logger.error('Expect: 100-continue on %s:%s%s' % (host, port, path)) return None
else: sleep(0.1) return resp except ChunkReadTimeout, err: self.logger.info("ChunkReadTimeout: %s" % err) return HTTPRequestTimeout(request=self.req) except (Exception, TimeoutError), err: self.logger.info("Error: %s" % err) return HTTPGatewayTimeout(request=self.req) else: try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, self.method, path, headers=self.headers, query_string=parsed.query, ssl=ssl) with Timeout(self.node_timeout): return conn.getresponse() except (Exception, TimeoutError), err: self.logger.debug("get response of GET or misc Error: %s" % err) return HTTPGatewayTimeout(request=self.req) class Dispatcher(object): """ """ def __init__(self, conf): self.conf = conf
self.logger.info('get response of PUT Error: %s' % err) if isinstance(resp, BufferedHTTPResponse): break else: sleep(0.1) return resp except ChunkReadTimeout, err: self.logger.info("ChunkReadTimeout: %s" % err) return HTTPRequestTimeout(request=self.req) except (Exception, TimeoutError), err: self.logger.info("Error: %s" % err) return HTTPGatewayTimeout(request=self.req) else: try: with ConnectionTimeout(self.conn_timeout): conn = http_connect_raw(host, port, self.method, path, headers=self.headers, query_string=parsed.query) with Timeout(self.node_timeout): return conn.getresponse() except (Exception, TimeoutError), err: self.logger.debug("get response of GET or misc Error: %s" % err) return HTTPGatewayTimeout(request=self.req) class Dispatcher(object): """ """ def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='dispatcher') self.dispatcher_addr = conf.get('bind_ip', '127.0.0.1') self.dispatcher_port = int(conf.get('bind_port', 8000)) self.ssl_enabled = True if 'cert_file' in conf else False self.relay_rule = conf.get('relay_rule')