def test_nonstr_header_values(self):
origHTTPSConnection = bufferedhttp.HTTPSConnection
bufferedhttp.HTTPSConnection = MockHTTPSConnection
try:
bufferedhttp.http_connect('127.0.0.1',
8080,
'sda',
1,
'GET',
'/',
headers={
'x-one': '1',
'x-two': 2,
'x-three': 3.0,
'x-four': {
'crazy': 'value'
}
},
ssl=True)
bufferedhttp.http_connect_raw('127.0.0.1',
8080,
'GET',
'/',
headers={
'x-one': '1',
'x-two': 2,
'x-three': 3.0,
'x-four': {
'crazy': 'value'
}
},
ssl=True)
finally:
bufferedhttp.HTTPSConnection = origHTTPSConnection
def test_nonstr_header_values(self):
with mock.patch('swift.common.bufferedhttp.HTTPSConnection',
MockHTTPSConnection):
bufferedhttp.http_connect('127.0.0.1',
8080,
'sda',
1,
'GET',
'/',
headers={
'x-one': '1',
'x-two': 2,
'x-three': 3.0,
'x-four': {
'crazy': 'value'
}
},
ssl=True)
bufferedhttp.http_connect_raw('127.0.0.1',
8080,
'GET',
'/',
headers={
'x-one': '1',
'x-two': 2,
'x-three': 3.0,
'x-four': {
'crazy': 'value'
}
},
ssl=True)
def test_nonstr_header_values(self):
class MockHTTPSConnection(object):
def __init__(self, hostport):
pass
def putrequest(self, method, path, skip_host=0):
pass
def putheader(self, header, *values):
# Essentially what Python 2.7 does that caused us problems.
'\r\n\t'.join(values)
def endheaders(self):
pass
origHTTPSConnection = bufferedhttp.HTTPSConnection
bufferedhttp.HTTPSConnection = MockHTTPSConnection
try:
bufferedhttp.http_connect('127.0.0.1', 8080, 'sda', 1, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
bufferedhttp.http_connect_raw('127.0.0.1', 8080, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
finally:
bufferedhttp.HTTPSConnection = origHTTPSConnection
def test_nonstr_header_values(self):
with mock.patch('swift.common.bufferedhttp.HTTPSConnection',
MockHTTPSConnection):
bufferedhttp.http_connect(
'127.0.0.1', 8080, 'sda', 1, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
bufferedhttp.http_connect_raw(
'127.0.0.1', 8080, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
def test_nonstr_header_values(self):
origHTTPSConnection = bufferedhttp.HTTPSConnection
bufferedhttp.HTTPSConnection = MockHTTPSConnection
try:
bufferedhttp.http_connect(
'127.0.0.1', 8080, 'sda', 1, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
bufferedhttp.http_connect_raw(
'127.0.0.1', 8080, 'GET', '/',
headers={'x-one': '1', 'x-two': 2, 'x-three': 3.0,
'x-four': {'crazy': 'value'}}, ssl=True)
finally:
bufferedhttp.HTTPSConnection = origHTTPSConnection
def GET(self, env, start_response):
"""
Handle GET Data Object request
"""
# Create a new WebOb Request object according to the current request
req = Request(env)
ssl = True if req.scheme.lower() == 'https' else False
conn = http_connect_raw(req.server_name, req.server_port, 'GET',
'/auth/v1.0', req.headers, None, ssl)
res = conn.getresponse()
# Create a new response
resp = Response()
if res.status == 200:
storage_url = res.getheader('X-Storage-Url')
resp.headers['X-Auth-Token'] = res.getheader('X-Auth-Token')
resp.headers['X-Storage-Token'] = res.getheader('X-Storage-Token')
subs = storage_url.partition('/v1/')
resp.headers['X-Storage-Url'] = (req.host_url + '/' +
self.cdmi_root + '/' + subs[2])
return resp
else:
resp.status = res.status
return resp
def _connect_put_node(self,
host,
port,
method,
path,
headers,
query_string,
ssl=False):
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host,
port,
method,
path,
headers=headers,
query_string=query_string,
ssl=ssl)
if headers.has_key('content-length') and int(
headers['content-length']) == 0:
return conn
with Timeout(self.node_timeout):
resp = conn.getexpect()
if resp.status == 100:
return conn
elif resp.status == 507:
self.logger.error('507 Insufficient Storage in %s:%s%s' %
(host, port, path))
raise Exception
except:
self.logger.error('Expect: 100-continue on %s:%s%s' %
(host, port, path))
return None
def GET(self, env, start_response):
"""
Handle GET Data Object request
"""
# Create a new WebOb Request object according to the current request
req = Request(env)
ssl = True if req.scheme.lower() == 'https' else False
conn = http_connect_raw(req.server_name, req.server_port, 'GET',
'/auth/v1.0', req.headers, None, ssl)
res = conn.getresponse()
# Create a new response
resp = Response()
if res.status == 200:
storage_url = res.getheader('X-Storage-Url')
resp.headers['X-Auth-Token'] = res.getheader('X-Auth-Token')
resp.headers['X-Storage-Token'] = res.getheader('X-Storage-Token')
subs = storage_url.partition('/v1/')
resp.headers['X-Storage-Url'] = (
req.host_url + '/' + self.cdmi_root + '/' + subs[2])
return resp
else:
resp.status = res.status
return resp
def direct_get_recon(node,
recon_command,
conn_timeout=5,
response_timeout=15,
headers=None):
"""
Get recon json directly from the storage server.
:param node: node dictionary from the ring
:param recon_command: recon string (post /recon/)
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param headers: dict to be passed into HTTPConnection headers
:returns: deserialized json response
:raises DirectClientReconException: HTTP GET request failed
"""
if headers is None:
headers = {}
ip, port = get_ip_port(node, headers)
path = '/recon/%s' % recon_command
with Timeout(conn_timeout):
conn = http_connect_raw(ip,
port,
'GET',
path,
headers=gen_headers(headers))
with Timeout(response_timeout):
resp = conn.getresponse()
if not is_success(resp.status):
raise DirectClientReconException('GET', node, path, resp)
return json.loads(resp.read())
def __call__(self):
"""
:return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
"""
if self.headers.has_key('content-length'):
if int(self.headers['content-length']) >= MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
parsed = urlparse(self.url)
if self.proxy:
proxy_parsed = urlparse(self.proxy)
if self._proxy_request_check(parsed.path):
host, port = self.split_netloc(proxy_parsed)
path = self.url
else:
host, port = self.split_netloc(parsed)
path = parsed.path
self.headers['host'] = '%s:%s' % (host, port)
if self.method == 'PUT' and len(parsed.path.split('/')) == 5:
chunked = self.req.headers.get('transfer-encoding')
reader = self.req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.chunk_size), '')
bytes_transferred = 0
# pile = GreenPile()
# pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query)
# conns = [conn for conn in pile if conn]
# conn = conns[0]
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host, port, self.method, path,
headers=self.headers, query_string=parsed.query)
with ContextPool(1) as pool:
conn.failed = False
conn.queue = Queue(10)
pool.spawn(self._send_file, conn, path)
while True:
with ChunkReadTimeout(self.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
conn.queue.put('0\r\n\r\n')
break
except TypeError, err:
self.logger.info('Chunk Read Error: %s' % err)
break
except Exception, err:
self.logger.info('Chunk Read Error: %s' % err)
return HTTPServerError(request=self.req)
bytes_transferred += len(chunk)
if bytes_transferred > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk)
def check_resource(env, method, path, logger, get_body=False,
query_string=None):
"""
Use this method to check if a resource already exist.
If the resource exists, then it should return True with headers.
If the resource does not exist, then it should return False with None
headers
If the get_body is set to True, the response body will also be returned
"""
# Create a new Request
req = Request(env)
ssl = True if req.scheme.lower() == 'https' else False
# Fixup the auth token, for some reason, the auth token padded the user
# account at the front with a comma. We need to get rid of it, otherwise,
# the auth token will be considered invalid.
headers = {}
if req.headers.get(Consts.AUTH_TOKEN):
key, sep, value = req.headers[Consts.AUTH_TOKEN].partition(',')
headers[Consts.AUTH_TOKEN] = value if value != '' else key
headers['Accept'] = 'application/json'
method = 'GET' if not method else method
path = req.path if not path else path
path = path.rstrip('/')
conn = http_connect_raw(req.server_name, req.server_port, method, path,
headers, query_string, ssl)
res = conn.getresponse()
if res.status == 404:
conn.close()
return False, {}, None
elif res.status == 200 or res.status == 204:
values = {}
header_list = res.getheaders()
for header in header_list:
values[header[0]] = header[1]
if get_body:
length = res.getheader('content-length')
if length:
body = res.read(int(length))
else:
body = res.read()
else:
body = ""
conn.close()
return True, values, body
else:
values = {}
header_list = res.getheaders()
for header in header_list:
values[header[0]] = header[1]
conn.close()
return True, values, None
def test_nonstr_header_values(self):
class MockHTTPSConnection(object):
def __init__(self, hostport):
pass
def putrequest(self, method, path, skip_host=0):
pass
def putheader(self, header, *values):
# Essentially what Python 2.7 does that caused us problems.
"\r\n\t".join(values)
def endheaders(self):
pass
origHTTPSConnection = bufferedhttp.HTTPSConnection
bufferedhttp.HTTPSConnection = MockHTTPSConnection
try:
bufferedhttp.http_connect(
"127.0.0.1",
8080,
"sda",
1,
"GET",
"/",
headers={"x-one": "1", "x-two": 2, "x-three": 3.0, "x-four": {"crazy": "value"}},
ssl=True,
)
bufferedhttp.http_connect_raw(
"127.0.0.1",
8080,
"GET",
"/",
headers={"x-one": "1", "x-two": 2, "x-three": 3.0, "x-four": {"crazy": "value"}},
ssl=True,
)
finally:
bufferedhttp.HTTPSConnection = origHTTPSConnection
def _connect_put_node(self, host, port, method, path, headers, query_string, ssl=False):
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host, port, method, path,
headers=headers, query_string=query_string,
ssl=ssl)
if headers.has_key('content-length') and int(headers['content-length']) == 0:
return conn
with Timeout(self.node_timeout):
resp = conn.getexpect()
if resp.status == 100:
return conn
elif resp.status == 507:
self.logger.error('507 Insufficient Storage in %s:%s%s' % (host, port, path))
raise Exception
except:
self.logger.error('Expect: 100-continue on %s:%s%s' % (host, port, path))
return None
else:
sleep(0.1)
return resp
except ChunkReadTimeout, err:
self.logger.info("ChunkReadTimeout: %s" % err)
return HTTPRequestTimeout(request=self.req)
except (Exception, TimeoutError), err:
self.logger.info("Error: %s" % err)
return HTTPGatewayTimeout(request=self.req)
else:
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host,
port,
self.method,
path,
headers=self.headers,
query_string=parsed.query,
ssl=ssl)
with Timeout(self.node_timeout):
return conn.getresponse()
except (Exception, TimeoutError), err:
self.logger.debug("get response of GET or misc Error: %s" %
err)
return HTTPGatewayTimeout(request=self.req)
class Dispatcher(object):
""" """
def __init__(self, conf):
self.conf = conf
self.logger.info('get response of PUT Error: %s' % err)
if isinstance(resp, BufferedHTTPResponse):
break
else:
sleep(0.1)
return resp
except ChunkReadTimeout, err:
self.logger.info("ChunkReadTimeout: %s" % err)
return HTTPRequestTimeout(request=self.req)
except (Exception, TimeoutError), err:
self.logger.info("Error: %s" % err)
return HTTPGatewayTimeout(request=self.req)
else:
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host, port, self.method, path,
headers=self.headers, query_string=parsed.query)
with Timeout(self.node_timeout):
return conn.getresponse()
except (Exception, TimeoutError), err:
self.logger.debug("get response of GET or misc Error: %s" % err)
return HTTPGatewayTimeout(request=self.req)
class Dispatcher(object):
""" """
def __init__(self, conf):
self.conf = conf
self.logger = get_logger(conf, log_route='dispatcher')
self.dispatcher_addr = conf.get('bind_ip', '127.0.0.1')
self.dispatcher_port = int(conf.get('bind_port', 8000))
self.ssl_enabled = True if 'cert_file' in conf else False
self.relay_rule = conf.get('relay_rule')
def __call__(self):
"""
:return httplib.HTTP(S)Connection in success, and webob.exc.HTTPException in failure
"""
if self.headers.has_key('content-length'):
if int(self.headers['content-length']) >= MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
parsed = urlparse(self.url)
if self.proxy:
proxy_parsed = urlparse(self.proxy)
if self._proxy_request_check(parsed.path):
host, port = self.split_netloc(proxy_parsed)
path = self.url
else:
host, port = self.split_netloc(parsed)
path = parsed.path
self.headers['host'] = '%s:%s' % (host, port)
if self.method == 'PUT' and len(parsed.path.split('/')) == 5:
chunked = self.req.headers.get('transfer-encoding')
reader = self.req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.chunk_size), '')
bytes_transferred = 0
# pile = GreenPile()
# pile.spawn(self._connect_server, host, port, self.method, path, self.headers, parsed.query)
# conns = [conn for conn in pile if conn]
# conn = conns[0]
try:
with ConnectionTimeout(self.conn_timeout):
conn = http_connect_raw(host,
port,
self.method,
path,
headers=self.headers,
query_string=parsed.query)
with ContextPool(1) as pool:
conn.failed = False
conn.queue = Queue(10)
pool.spawn(self._send_file, conn, path)
while True:
with ChunkReadTimeout(self.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
conn.queue.put('0\r\n\r\n')
break
except TypeError, err:
self.logger.info('Chunk Read Error: %s' % err)
break
except Exception, err:
self.logger.info('Chunk Read Error: %s' % err)
return HTTPServerError(request=self.req)
bytes_transferred += len(chunk)
if bytes_transferred > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=self.req)
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
