def index(self, req):
""" Handle GET and HEAD requests for static files. Directory requests are not allowed"""
static_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../static/'))
# filter out ..
try:
static_path = req.urlvars['path'].replace('/..', '')
except:
return HTTPForbidden()
path = os.path.join(static_dir, static_path)
if os.path.isdir(path):
return HTTPForbidden()
if req.method == 'GET' or req.method == 'HEAD':
if os.path.isfile(path):
etag, modified, mime_type, size = self._get_stat(path)
res = Response()
res.headers['content-type'] = mime_type
res.date = rfc822.formatdate(time.time())
res.last_modified = modified
res.etag = etag
if_modified_since = req.headers.get('HTTP_IF_MODIFIED_SINCE')
if if_modified_since:
if rfc822.parsedate(if_modified_since) >= rfc822.parsedate(last_modified):
return HTTPNotModified()
if_none_match = req.headers.get('HTTP_IF_NONE_MATCH')
if if_none_match:
if if_none_match == '*' or etag in if_none_match:
return HTTPNotModified()
# set the response body
if req.method == 'GET':
fd = open(path, 'rb')
if 'wsgi.file_wrapper' in req.environ:
res.app_iter = req.environ['wsgi.file_wrapper'](fd)
res.content_length = size
else:
res.app_iter = iter(lambda: fd.read(8192), '')
res.content_length = size
else:
res.body = ''
return res
else:
return None
else:
return None
def index(self, req):
""" Handle GET and HEAD requests for static files. Directory requests are not allowed"""
static_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '../static/'))
# filter out ..
try:
static_path = req.urlvars['path'].replace('/..', '')
except:
return HTTPForbidden()
path = os.path.join(static_dir, static_path)
if os.path.isdir(path):
return HTTPForbidden()
if req.method == 'GET' or req.method == 'HEAD':
if os.path.isfile(path):
etag, modified, mime_type, size = self._get_stat(path)
res = Response()
res.headers['content-type'] = mime_type
res.date = rfc822.formatdate(time.time())
res.last_modified = modified
res.etag = etag
if_modified_since = req.headers.get('HTTP_IF_MODIFIED_SINCE')
if if_modified_since:
if rfc822.parsedate(if_modified_since) >= rfc822.parsedate(last_modified):
return HTTPNotModified()
if_none_match = req.headers.get('HTTP_IF_NONE_MATCH')
if if_none_match:
if if_none_match == '*' or etag in if_none_match:
return HTTPNotModified()
# set the response body
if req.method == 'GET':
fd = open(path, 'rb')
if 'wsgi.file_wrapper' in req.environ:
res.app_iter = req.environ['wsgi.file_wrapper'](fd)
res.content_length = size
else:
res.app_iter = iter(lambda: fd.read(8192), '')
res.content_length = size
else:
res.body = ''
return res
else:
return None
else:
return None
def make_response(uri, environ):
res = Response(conditional_response=True)
# check if the host is online. If not raise an http error
if not pingSMB( parseSMBuri(uri)["host"] ):
return HTTPGatewayTimeout("Host is currently offline. You may try again at a later time.")
try:
f = c.open(uri)
except smbc.NoEntryError:
return HTTPNotFound("The file you requested is no longer available!")
fs = f.fstat()
filesize = fs[6]
last_modified = fs[8] # mtime
filename = uri.split("/")[-1]
req = Request(environ)
log.debug("Incoming request: \n" + str(req))
if req.range:
log.debug("begin ranged transfer")
res.status_int = 206
res.content_range = req.range.content_range(filesize)
(start, stop) = req.range.range_for_length(filesize)
log.debug("filesize: " + str(filesize))
log.debug("start: " + str(start) + " stop: " + str(stop))
log.debug("Content-Range: " + str(res.content_range))
res.app_iter = FileIterable(uri, start=start, stop=stop)
res.content_length = stop - start
else:
log.debug("begin normal transfer")
res.app_iter = FileIterable(uri)
res.content_length = filesize
log.debug("Content-Length: " + str(res.content_length))
res.server_protocol = "HTTP/1.1"
# Make sure the file gets downloaded and not played live
res.content_type = "application/octet-stream"
res.last_modified = last_modified
res.etag = '%s-%s-%s' % (fs[8], fs[6], hash(f))
res.headers.add("Content-Disposition", 'attachment; filename="%s"' % str(filename) )
res.headers.add("Accept-Ranges", "bytes")
return res
def app(environ, start_response):
"""Multipart AJAX request example.
See: http://test.getify.com/mpAjax/description.html
"""
response = Response()
parts = []
for i in range(12):
for j in range(12):
parts.append(executor.submit(mul, i, j))
def stream(parts, timeout=None):
try:
for future in as_completed(parts, timeout):
mime, result = future.result()
result = result.encode('utf8')
yield "!!!!!!=_NextPart_{num}\nContent-Type: {mime}\nContent-Length: {length}\n\n".format(
num=randint(100000000, 999999999),
mime=mime,
length=len(result)).encode('utf8') + result
except TimeoutError:
for future in parts:
future.cancel()
response.content_length = None
response.app_iter = stream(parts, 0.2)
return response(environ, start_response)
def do_download(self, collection_id, post_data, is_new=False):
if is_new:
return self.error_response('POST argument required: collection_id')
writer = post_data.get('writer', self.default_writer)
try:
log.info('download %s %s' % (collection_id, writer))
output_path = self.get_path(collection_id, self.output_filename, writer)
os.utime(output_path, None)
status = self.read_status_file(collection_id, writer)
response = Response()
response.app_iter = FileIterable(output_path)
response.content_length = os.path.getsize(output_path)
if 'content_type' in status:
response.content_type = status['content_type'].encode('utf-8', 'ignore')
else:
log.warn('no content type in status file')
if 'file_extension' in status:
response.headers['Content-Disposition'] = 'inline; filename=collection.%s' % (
status['file_extension'].encode('utf-8', 'ignore'),
)
else:
log.warn('no file extension in status file')
return response
except Exception, exc:
log.ERROR('exception in do_download(): %r' % exc)
return Response(status=500)
def test_HEAD_conditional_response_range_empty_response():
from webob.response import EmptyResponse
req = Request.blank('/')
req.method = 'HEAD'
res = Response(request=req, conditional_response=True)
res.status_int = 200
res.body = 'Are we not men?'
res.content_length = len(res.body)
class FakeRequest:
method = 'HEAD'
if_none_match = 'none'
if_modified_since = False
def __init__(self, env):
self.env = env
self.range = self # simulate inner api
self.if_range = self
def content_range(self, length):
"""range attr"""
class Range:
start = 4
stop = 5
return Range
def match_response(self, res):
"""if_range_match attr"""
return True
def start_response(status, headerlist):
pass
res.RequestClass = FakeRequest
result = res({}, start_response)
ok_(isinstance(result, EmptyResponse), result)
def app(environ, start_response):
"""Multipart AJAX request example.
See: http://test.getify.com/mpAjax/description.html
"""
response = Response()
parts = []
for i in range(12):
for j in range(12):
parts.append(executor.submit(mul, i, j))
def stream(parts, timeout=None):
try:
for future in as_completed(parts, timeout):
mime, result = future.result()
result = result.encode('utf8')
yield "!!!!!!=_NextPart_{num}\nContent-Type: {mime}\nContent-Length: {length}\n\n".format(
num = randint(100000000, 999999999),
mime = mime,
length = len(result)
).encode('utf8') + result
except TimeoutError:
for future in parts:
future.cancel()
response.content_length = None
response.app_iter = stream(parts, 0.2)
return response(environ, start_response)
def test_app_iter_del():
res = Response()
res.content_length = 3
res._app_iter = ['123']
del res.app_iter
eq_(res._app_iter, None)
eq_(res._body, None)
eq_(res.content_length, None)
def __call__(self, environ, start_response):
res = Response(conditional_response=True)
def x():
for i in xrange(10):
yield str(i)
res.app_iter = x()
res.content_length = 10
return res(environ, start_response)
def make_response(file, content_type):
response = Response(content_type=content_type)
response.app_iter = FileIterable(file)
stat = os.fstat(file.fileno())
response.content_length = stat.st_size
response.last_modified = stat.st_mtime
response.etag = '%s-%s' % (stat.st_mtime, stat.st_size)
return response
def make_response(filename):
res = Response(content_type=get_mimetype(filename))
res.app_iter = FileIterable(filename)
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename), hash(filename))
return res
def test_app_iter_del():
res = Response()
res.content_length = 3
res._app_iter = ['123']
del res.app_iter
eq_(res._app_iter, None)
eq_(res._body, None)
eq_(res.content_length, None)
def make_response(filename):
res = Response(content_type=get_mimetype(filename),
conditional_response=True)
res.app_iter = FileIterable(filename)
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename), hash(filename))
return res
def test_unicode_body_del():
res = Response()
res._body = '123'
res.content_length = 3
res._app_iter = ()
del res.unicode_body
eq_(res._body, None)
eq_(res.content_length, None)
eq_(res._app_iter, None)
def test_body_file_del():
res = Response()
res._body = '123'
res.content_length = 3
res._app_iter = ()
del res.body_file
eq_(res._body, None)
eq_(res.content_length, None)
eq_(res._app_iter, None)
def test_unicode_body_del():
res = Response()
res._body = '123'
res.content_length = 3
res._app_iter = ()
del res.unicode_body
eq_(res._body, None)
eq_(res.content_length, None)
eq_(res._app_iter, None)
def test_body_file_del():
res = Response()
res._body = '123'
res.content_length = 3
res._app_iter = ()
del res.body_file
eq_(res._body, None)
eq_(res.content_length, None)
eq_(res._app_iter, None)
def file_response(filename):
"""return a webob response object appropriate to a file name"""
res = Response(content_type=get_mimetype(filename),
conditional_response=True)
res.app_iter = FileIterable(filename)
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename), hash(filename))
return res
def __call__(self, environ, start_response):
res = Response(conditional_response=True)
def x():
for i in xrange(10):
yield str(i)
res.app_iter = x()
res.content_length = 10
return res(environ, start_response)
def do_thumb(self, req, image):
"""
Serve a thumbnail of an image from the library
"""
if not image in self.library:
self.not_found(req)
resp = Response()
resp.content_type = 'image/jpeg'
resp.content_length = self.library.stat_thumbnail(image).st_size
resp.app_iter = FileWrapper(self.library.open_thumbnail(image))
return resp
def do_thumb(self, req, image):
"""
Serve a thumbnail of an image from the library
"""
if not image in self.library:
self.not_found(req)
resp = Response()
resp.content_type = 'image/jpeg'
resp.content_length = self.library.stat_thumbnail(image).st_size
resp.app_iter = FileWrapper(self.library.open_thumbnail(image))
return resp
def do_image(self, req, image):
"""
Serve an image from the library
"""
if not image in self.library:
self.not_found(req)
resp = Response()
resp.content_type, resp.content_encoding = mimetypes.guess_type(
image, strict=False)
resp.content_length = self.library.stat_image(image).st_size
resp.app_iter = FileWrapper(self.library.open_image(image))
return resp
def do_image(self, req, image):
"""
Serve an image from the library
"""
if not image in self.library:
self.not_found(req)
resp = Response()
resp.content_type, resp.content_encoding = mimetypes.guess_type(
image, strict=False)
resp.content_length = self.library.stat_image(image).st_size
resp.app_iter = FileWrapper(self.library.open_image(image))
return resp
def make_file_response(storage, filename, range=None):
try:
res = Response(content_type=get_mimetype(filename),
conditional_response=True)
res.app_iter = FileIterable(storage, filename)
res.content_length = storage.size(filename)
res.last_modified = storage.modified_time(filename)
res.etag = '%s-%s-%s' % (storage.modified_time(filename),
storage.size(filename), hash(filename))
return res
except OSError:
return HTTPNotFound("Not found")
def do_download(self, req):
"""
Send the library as a .zip archive
"""
archive = self.library.archive()
size = archive.seek(0, io.SEEK_END)
archive.seek(0)
resp = Response()
resp.content_type = 'application/zip'
resp.content_length = size
resp.content_disposition = 'attachment; filename=images.zip'
resp.app_iter = FileWrapper(archive)
return resp
def send_file(self, path, mimetype):
"""Send the file located at 'path' back to the user
"""
response = Response(content_type=mimetype, conditional_response=True)
response.last_modified = os.path.getmtime(path)
response.app_iter = FileIterable(path)
with open(path) as f:
response.body = f.read()
response.content_length = os.path.getsize(path)
# do not accept ranges, since this does not work reliable
# with acrobat IE plugin
response.headers['Accept-Ranges'] = 'none'
return response
def do_download(self, req):
"""
Send the library as a .zip archive
"""
archive = self.library.archive()
size = archive.seek(0, io.SEEK_END)
archive.seek(0)
resp = Response()
resp.content_type = 'application/zip'
resp.content_length = size
resp.content_disposition = 'attachment; filename=images.zip'
resp.app_iter = FileWrapper(archive)
return resp
def make_response(filename, if_iter = True, if_ranger = False):
if not if_iter:
res = Response(content_type = get_mimetype(filename))
res.body = open(filename, 'rb').read()
else:
res = Response(content_type = get_mimetype(filename),
conditional_response = True)
res.app_iter = FileIterable(filename)
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename),
hash(filename))
return res
def send_file(self, path, mimetype):
"""Send the file located at 'path' back to the user
"""
response = Response(content_type=mimetype,
conditional_response=True)
response.last_modified = os.path.getmtime(path)
response.app_iter = FileIterable(path)
with open(path) as f:
response.body = f.read()
response.content_length = os.path.getsize(path)
# do not accept ranges, since this does not work reliable
# with acrobat IE plugin
response.headers['Accept-Ranges'] = 'none'
return response
def do_static(self, req, path):
"""
Serve static files from disk
"""
path = os.path.normpath(os.path.join(self.static_dir, path))
if not path.startswith(self.static_dir):
self.not_found(req)
resp = Response()
resp.content_type, resp.content_encoding = mimetypes.guess_type(
path, strict=False)
if resp.content_type is None:
resp.content_type = 'application/octet-stream'
resp.content_length = os.stat(path).st_size
resp.app_iter = FileWrapper(io.open(path, 'rb'))
return resp
def do_static(self, req, path):
"""
Serve static files from disk
"""
path = os.path.normpath(os.path.join(self.static_dir, path))
if not path.startswith(self.static_dir):
self.not_found(req)
resp = Response()
resp.content_type, resp.content_encoding = mimetypes.guess_type(
path, strict=False)
if resp.content_type is None:
resp.content_type = 'application/octet-stream'
resp.content_length = os.stat(path).st_size
resp.app_iter = FileWrapper(io.open(path, 'rb'))
return resp
def sendRawResponse(status, filename, lastmod):
"""Send data. Assume status is a number and filename is the name of a file
containing the body of the response."""
resp = Response(status=status, content_type='text/html')
resp.headers['Access-Control-Allow-Origin'] = '*'
if lastmod and status != 304:
resp.last_modified = lastmod
fp = open(filename)
fp.seek(0, 2)
size = fp.tell()
fp.seek(0)
resp.content_length = size
resp.app_iter = Chunked(fp)
return resp
def sendRawResponse(status, filename, lastmod):
"""Send data. Assume status is a number and filename is the name of a file
containing the body of the response."""
resp = Response(status=status, content_type='text/html')
resp.headers['Access-Control-Allow-Origin'] = '*'
if lastmod and status != 304:
resp.last_modified = lastmod
fp = open(filename)
fp.seek(0, 2)
size = fp.tell()
fp.seek(0)
resp.content_length = size
resp.app_iter = Chunked(fp)
return resp
def application(env, start_response):
res = None
path = env['PATH_INFO'].lstrip('/')
method = env['REQUEST_METHOD']
args = MultiDict(parse_qsl(env.get('QUERY_STRING', '')))
parts = path.split('/')
volume = args.get('volume')
if not volume:
volumes = cluster.get_volumes(args['key'])
for v in volumes:
if v.node is current_node and v.id in current_node.volumes:
volume = v.id
break
if volume:
v = current_node.volumes[volume]
collection = parts[0]
if method == 'PUT':
return put_file(env, start_response,
v, collection, args['key'])
elif method == 'GET':
fname, meta = v.get(collection, args['key'])
res = Response()
res.charset = None
res.headers['X-Sendfile'] = fname
res.headers['X-Crc'] = str(meta['crc'])
res.content_length = os.path.getsize(fname)
if 'ct' in meta:
res.content_type = meta['ct']
else:
collection = parts[0]
key = args['key']
v = volumes[0]
res = HTTPTemporaryRedirect(
location='http://{}/{}?key={}&volume={}'.format(v.node.id, collection, key, v.id))
for v in volumes:
res.headers.add(
'X-Location',
'http://{}/{}?key={}&volume={}'.format(v.node.id, collection, key, v.id))
if not res:
res = HTTPNotFound()
return res(env, start_response)
def make_folder_response(foldername, req):
if req.path_url.endswith('/'):
fold_url = req.path_url[:-1]
else:
fold_url = req.path_url
folderlist = [ '<html>' + #foldername +
'<br>'.join( [('<a href="%s/%s">%s</a>' % (fold_url, filename, filename))
for filename in os.listdir(foldername)])
+ '</html>' ]
res = Response(content_type="text/html",
conditional_response=True)
res.body = folderlist[0]
res.headers.add('Server', render('__server_info__'))
res.content_length = len(res.body)
res.last_modified = os.path.getmtime(foldername)
res.etag = '%s-%s-%s' % (os.path.getmtime(foldername),
os.path.getsize(foldername), hash(foldername))
return res
def serve_file(filename):
if os.path.exists(filename):
basename = urlutils.basename(filename)
content_type = mimetypes.guess_type(basename)[0]
res = Response(content_type=content_type, conditional_response=True)
res.app_iter = FileIterable(filename)
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
# Todo: is this the best value for the etag?
# perhaps md5 would be a better alternative
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename),
hash(filename))
return res
else:
return HTTPNotFound()
def dispatch(self, request):
"""Returns a response if a file can be found in the public directory of one of the rambler extension
otherwise it returns None"""
path = 'public/' + request.path_info
if path.endswith('/'):
path += 'index.html'
path = os.path.normpath(path)
for module in self.mod_names:
try:
stream = pkg_resources.resource_stream(module + '.web_controllers', path )
response = Response(request=request, app_iter = stream)
response.content_type = mimetypes.guess_type(stream.name)[0]
response.content_length = os.path.getsize(stream.name)
return response
except (IOError, ImportError):
pass
def make_file_response(filename):
res = Response(content_type=get_mimetype(filename),
conditional_response=True)
res.headers.add('Accept-Ranges','bytes')
res.headers.add('Server', render('__server_info__'))
res.headers.add('Content-Disposition',str('attachment; filename=%s'%(filename.split(os.path.sep)[-1])))
res.app_iter = FileIterable(filename)
#try:
res.content_length = os.path.getsize(filename)
res.last_modified = os.path.getmtime(filename)
res.etag = '%s-%s-%s' % (os.path.getmtime(filename),
os.path.getsize(filename), hash(filename))
#===========================================================================
# except WindowsError, e:
# if e.errno == 2:
# del res
# res = Response(status='404')
# res.headers.add('Server', render('__server_info__'))
#===========================================================================
return res
def _handle_request(self, request):
try:
outgoing = self.router.route(request).prepare()
except NoRouteError:
raise HTTPNotFound()
except Exception:
raise
raise HTTPBadGateway()
else:
upstream = self.requests.send(outgoing, stream=True,
proxies=self.proxies)
response = Response(status=upstream.status_code, headers={})
print upstream.headers
for header, value in upstream.headers.items():
print "ADD HEADER", header
if is_hop_by_hop(header):
continue
response.headers.add(capitalize_header(header),
value)
clen = response.headers.get('Content-Length')
response.app_iter = upstream.iter_content(4096)
response.content_length = clen
return response
def _handle_request(self, request):
try:
outgoing = self.router.route(request).prepare()
except NoRouteError:
raise HTTPNotFound()
except Exception:
raise
raise HTTPBadGateway()
else:
upstream = self.requests.send(outgoing,
stream=True,
proxies=self.proxies)
response = Response(status=upstream.status_code, headers={})
print upstream.headers
for header, value in upstream.headers.items():
print "ADD HEADER", header
if is_hop_by_hop(header):
continue
response.headers.add(capitalize_header(header), value)
clen = response.headers.get('Content-Length')
response.app_iter = upstream.iter_content(4096)
response.content_length = clen
return response
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: webob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: webob.Response object
"""
statuses = []
reasons = []
bodies = []
source = None
sources = []
newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'], node['port'],
node['device'], partition, req.method, path,
headers=headers,
query_string=req.query_string)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(node, server_type,
_('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
continue
if is_success(possible_source.status) or \
is_redirection(possible_source.status):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
possible_source.read()
continue
if newest:
if sources:
ts = float(source.getheader('x-put-timestamp') or
source.getheader('x-timestamp') or 0)
pts = float(
possible_source.getheader('x-put-timestamp') or
possible_source.getheader('x-timestamp') or 0)
if pts > ts:
sources.insert(0, possible_source)
else:
sources.append(possible_source)
else:
sources.insert(0, possible_source)
source = sources[0]
statuses.append(source.status)
reasons.append(source.reason)
bodies.append('')
continue
else:
source = possible_source
break
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024], 'type': server_type})
if source:
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
if newest:
# we need to close all hanging swift_conns
sources.pop(0)
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
res.app_iter = self._make_app_iter(node, source, res)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.status = source.status
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
elif is_success(source.status) or is_redirection(source.status):
res = status_map[source.status](request=req)
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
if if_modified_since and \
datetime.fromtimestamp(float(file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=file,
request=request, conditional_response=True)
response.headers['Content-Type'] = file.metadata.get('Content-Type',
'application/octet-stream')
for key, value in file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = file.metadata['ETag']
response.last_modified = float(file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < KEEP_CACHE_SIZE and \
'X-Auth-Token' not in request.headers and \
'X-Storage-Token' not in request.headers:
file.keep_cache = True
if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
return request.get_response(response)
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
except ValueError, err:
return response
if "range" in req.headers:
status_int = httplib.PARTIAL_CONTENT
response_headers["Content-Range"] = \
_content_range_header(lower_bound,
upper_bound,
retriever.total_file_size)
content_length = slice_size
else:
status_int = httplib.OK
# 2012-09-06 dougfort Ticket #44 (temporary Connection: close)
response.headers["Connection"] = "close"
response.last_modified = last_modified
response.content_length = content_length
if content_type is None:
response.content_type = "application/octet-stream"
else:
response.content_type = content_type
if content_encoding is not None:
response.content_encoding = content_encoding
response.status_int = status_int
response.app_iter = retrieve_generator
return response
def _retrieve_meta(self, req, match_object, user_request_id):
collection_name = match_object.group("collection_name")
response.status_int = httplib.PRECONDITION_FAILED
return response
if "range" in req.headers:
status_int = httplib.PARTIAL_CONTENT
response_headers["Content-Range"] = _content_range_header(
lower_bound, upper_bound, retriever.total_file_size
)
content_length = slice_size
else:
status_int = httplib.OK
# 2012-09-06 dougfort Ticket #44 (temporary Connection: close)
response.headers["Connection"] = "close"
response.last_modified = last_modified
response.content_length = content_length
if content_type is None:
response.content_type = "application/octet-stream"
else:
response.content_type = content_type
if content_encoding is not None:
response.content_encoding = content_encoding
response.status_int = status_int
response.app_iter = retrieve_generator
return response
def _retrieve_meta(self, req, match_object, user_request_id):
collection_name = match_object.group("collection_name")
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse(
(parsed_req_url.scheme, relay_addr + ':' + relay_port,
'/' + '/'.join(path_str_ls), parsed_req_url.params,
parsed_req_url.query, parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
self.logger.debug('Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
result = RelayRequest(self.conf,
req,
connect_url,
proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info('Retry Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
continue
else:
return result
response = Response(status='%s %s' %
(result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(
response,
{'Content-Length': result.getheader('Content-Length')})
response.status = result.status
return response
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse((parsed_req_url.scheme,
relay_addr + ':' + relay_port,
'/' + '/'.join(path_str_ls),
parsed_req_url.params,
parsed_req_url.query,
parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
self.logger.debug('Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
result = RelayRequest(self.conf, req, connect_url, proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info('Retry Req: %s %s, Connect to %s via %s' %
(req.method, req.url, connect_url, proxy))
continue
else:
return result
response = Response(status='%s %s' % (result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(response, {'Content-Length':
result.getheader('Content-Length')})
response.status = result.status
return response
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: webob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: webob.Response object
"""
statuses = []
reasons = []
bodies = []
source = None
sources = []
newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'],
node['port'],
node['device'],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type,
_('Trying to %(method)s %(path)s') % {
'method': req.method,
'path': req.path
})
continue
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
continue
if is_success(possible_source.status) or \
is_redirection(possible_source.status):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
possible_source.read()
continue
if newest:
if sources:
ts = float(
source.getheader('x-put-timestamp')
or source.getheader('x-timestamp') or 0)
pts = float(
possible_source.getheader('x-put-timestamp')
or possible_source.getheader('x-timestamp') or 0)
if pts > ts:
sources.insert(0, possible_source)
else:
sources.append(possible_source)
else:
sources.insert(0, possible_source)
source = sources[0]
statuses.append(source.status)
reasons.append(source.reason)
bodies.append('')
continue
else:
source = possible_source
break
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024], 'type': server_type})
if source:
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
if newest:
# we need to close all hanging swift_conns
sources.pop(0)
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.status = source.status
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
elif is_success(source.status) or is_redirection(source.status):
res = status_map[source.status](request=req)
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
except FutureRevision, e:
error = "<h1>No such revision %s</h1>" % e.rev
last_changed_rev = self.svn(request).last_changed_rev(
request.path_info, version)
request.GET['version'] = last_changed_rev
redirected = redirect(request)
error += "<div>Perhaps try <a href='%s'>its last revision</a> instead" % redirected.location
return exc.HTTPNotFound(body=error)
contents = {'contents': data}
mimetype = "text/html"
view = self.viewer.match_view(request, data, mimetype)
if view:
return view(request, data)
contents['prev_href'] = "%s?version=%d" % (
request.path_info, int(request.GET['version']) - 1)
x = self.svn(request)
uri = request.path_info
contents['format'] = request.GET.get('format')
res = Response(content_type=mimetype, body=data)
res.content_length = len(res.body)
res.headers['X-Svenweb-Version'] = version
return res
last_changed_rev = self.svn(request).last_changed_rev(request.path_info, version)
request.GET['version'] = last_changed_rev
redirected = redirect(request)
error += "<div>Perhaps try <a href='%s'>its last revision</a> instead" % redirected.location
return exc.HTTPNotFound(body=error)
contents = {'contents': data}
mimetype = "text/html"
view = self.viewer.match_view(request, data, mimetype)
if view:
return view(request, data)
contents['prev_href'] = "%s?version=%d" % (request.path_info,
int(
request.GET['version']) - 1)
x = self.svn(request)
uri = request.path_info
contents['format'] = request.GET.get('format')
res = Response(
content_type=mimetype,
body=data)
res.content_length = len(res.body)
res.headers['X-Svenweb-Version'] = version
return res
datetime.fromtimestamp(float(file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=file,
request=request,
conditional_response=True)
response.headers['Content-Type'] = file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = file.metadata['ETag']
response.last_modified = float(file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < KEEP_CACHE_SIZE and \
'X-Auth-Token' not in request.headers and \
'X-Storage-Token' not in request.headers:
file.keep_cache = True
if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = file.metadata['X-Timestamp']
return request.get_response(response)
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Chase Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
except ValueError, err:
def relay_req(self, req, req_url, path_str_ls, relay_servers, webcaches):
""" """
# util
def get_relay_netloc(relay_server):
parsed = urlparse(relay_server)
svr = parsed.netloc.split(':')
if len(svr) == 1:
relay_addr = svr[0]
relay_port = '443' if parsed.scheme == 'https' else '80'
else:
relay_addr, relay_port = svr
return relay_addr, relay_port
relay_id = str(uuid4())
parsed_req_url = urlparse(req_url)
relay_servers_count = len(relay_servers)
connect_path = '/' + '/'.join(path_str_ls)
if parsed_req_url.path.endswith('/'):
connect_path = connect_path + '/'
for relay_server in relay_servers:
relay_addr, relay_port = get_relay_netloc(relay_server)
connect_url = urlunparse(
(parsed_req_url.scheme, relay_addr + ':' + relay_port,
connect_path, parsed_req_url.params, parsed_req_url.query,
parsed_req_url.fragment))
if webcaches[relay_server]:
proxy = webcaches[relay_server]
else:
proxy = None
if req.headers.has_key('x-object-manifest'):
object_manifest = req.headers['x-object-manifest']
cont = object_manifest.split('/')[0]
obj = object_manifest.split('/')[1:]
cont_parts = cont.split(':')
if len(cont_parts) >= 2:
real_cont = ':'.join(cont_parts[1:])
object_manifest = real_cont + '/' + '/'.join(obj)
req.headers['x-object-manifest'] = object_manifest
original_url = req.url
self.logger.info(
'Request[%s]: %s %s with headers = %s, Connect to %s (via %s)'
% (str(relay_id), req.method, req.url, req.headers,
connect_url, proxy))
result = RelayRequest(self.conf,
req,
connect_url,
proxy=proxy,
conn_timeout=self.conn_timeout,
node_timeout=self.node_timeout,
chunk_size=self.client_chunk_size)()
if isinstance(result, HTTPException):
if relay_servers_count > 1:
relay_servers_count -= 1
self.logger.info(
'Retry Req[%s]: %s %s with headers = %s, Connect to %s (via %s)'
% (str(relay_id), req.method, req.url, req.headers,
connect_url, proxy))
continue
else:
return result
if result.getheader('location'):
location = result.getheader('location')
parsed_location = urlparse(location)
parsed_connect_url = urlparse(connect_url)
if parsed_location.netloc.startswith(
parsed_connect_url.netloc):
parsed_orig_url = urlparse(original_url)
loc_prefix = parsed_orig_url.path.split('/')[1]
if parsed_orig_url.path.split(
'/')[1] != self.req_version_str:
rewrited_path = '/' + loc_prefix + parsed_location.path
else:
rewrited_path = parsed_location.path
rewrited_location = (parsed_orig_url.scheme,
parsed_orig_url.netloc, rewrited_path,
parsed_location.params,
parsed_location.query,
parsed_location.fragment)
response = Response(status='%s %s' %
(result.status, result.reason))
response.bytes_transferred = 0
def response_iter():
try:
while True:
with ChunkReadTimeout(self.client_timeout):
chunk = result.read(self.client_chunk_size)
if not chunk:
break
yield chunk
response.bytes_transferred += len(chunk)
except GeneratorExit:
pass
except (Exception, TimeoutError):
raise
response.headerlist = result.getheaders()
response.content_length = result.getheader('Content-Length')
if response.content_length < 4096:
response.body = result.read()
else:
response.app_iter = response_iter()
update_headers(response, {'accept-ranges': 'bytes'})
response.content_length = result.getheader('Content-Length')
update_headers(response, result.getheaders())
if req.method == 'HEAD':
update_headers(
response,
{'Content-Length': result.getheader('Content-Length')})
if result.getheader('location'):
update_headers(response,
{'Location': urlunparse(rewrited_location)})
response.status = result.status
self.logger.info('Response[%s]: %s by %s %s %s' %
(str(relay_id), response.status, req.method,
req.url, response.headers))
return response
def __call__(self, environ, start_response):
res = Response(conditional_response=True)
res.body = ''.join(str(i) for i in xrange(10))
res.content_length = 10
return res(environ, start_response)
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(float(file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
file.close()
return HTTPNotModified(request=request)
response = Response(content_type=file.metadata.get('Content-Type',
'application/octet-stream'), app_iter=file,
request=request, conditional_response=True)
for key, value in file.metadata.iteritems():
if key == 'X-Object-Manifest' or \
key.lower().startswith('x-object-meta-'):
response.headers[key] = value
response.etag = file.metadata['ETag']
response.last_modified = float(file.metadata['X-Timestamp'])
response.content_length = int(file.metadata['Content-Length'])
if response.content_length < KEEP_CACHE_SIZE and \
'X-Auth-Token' not in request.headers and \
'X-Storage-Token' not in request.headers:
file.keep_cache = True
if 'Content-Encoding' in file.metadata:
response.content_encoding = file.metadata['Content-Encoding']
return request.get_response(response)
def HEAD(self, request):
"""Handle HTTP HEAD requests for the Swift Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
except ValueError, err:
resp = HTTPBadRequest(request=request)
