def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = jresponse('-1', 'invalid utf8', req,412)
else:
try:
# disallow methods which are not publicly accessible
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = jresponse('-1', 'method not allowed', req,405)
else:
res = method(req)
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = jresponse('-1', 'InternalServerError', req,500)
additional_info = ''
if res.headers.get('x-container-timestamp') is not None:
additional_info += 'x-container-timestamp: %s' % \
res.headers['x-container-timestamp']
return res(env, start_response)
def check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
:param req: HTTP request object
:param object_name: name of object to be created
:raises HTTPRequestEntityTooLarge: the object is too large
:raises HTTPLengthRequered: missing content-length header and not
a chunked request
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
if req.content_length and req.content_length > MAX_FILE_SIZE:
respbody='Your request is too large.'
return jresponse('-1', respbody, req,413)
if req.content_length is None and \
req.headers.get('transfer-encoding') != 'chunked':
return jresponse('-1', 'length required', req,411)
if len(object_name) > MAX_OBJECT_NAME_LENGTH:
respbody='Object name length of %d longer than %d' % (len(object_name), MAX_OBJECT_NAME_LENGTH)
return jresponse('-1', respbody, req,400)
return None
def __call__(self, request):
if request.method not in ("PUT","COPY"):
return self.app
try:
split_path(request.path,2, 4, rest_with_last=True)
except ValueError:
return self.app
new_quota = request.headers.get('X-Account-Meta-Quota-Bytes')
if new_quota:
if not new_quota.isdigit():
return jresponse('-1', 'bad request', request, 400)
return self.app
account_info = get_account_info(request.environ, self.app)
new_size = int(account_info['bytes']) + (request.content_length or 0)
quota = int(account_info['meta'].get('quota-bytes', -1))
if 0 <= quota < new_size:
respbody='Your request is too large.'
return jresponse('-1', respbody, request,413)
return self.app
def MOVE(self,req):
if 'ftype' not in req.GET:
return jresponse('-1', 'param error', req,404)
(container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if not containers:
return jresponse('-1', 'not found', req,404)
direr_partition, direr_nodes = self.app.direr_ring.get_nodes(self.account_name, self.container_name, self.direr_name)
headers = []
for container in containers:
nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Container-Host': '%(ip)s:%(port)s' % container,
'X-Container-Partition': container_partition,
'X-Container-Device': container['device'],
'x-move-dst':req.headers['Destination'],
'x-ftype':req.GET['ftype'],
'Connection': 'close'}
if object_versions:
nheaders['x-versions-location'] = object_versions
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.direr_ring,
direr_partition, 'MOVE', req.path_info, headers)
return resp
def __call__(self, env, start_response):
"""WSGI Application entry point for the Swift Object Server."""
start_time = time.time()
req = Request(env)
if 'l' != req.headers.get('X-Ftype'):
return self.app(env,start_response)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res =jresponse('-1', 'Invalid UTF8', req,412)
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = jresponse('-1', 'method not allowed', req,405)
else:
res = method(req)
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = jresponse('-1', 'InternalServerError', req,500)
trans_time = time.time() - start_time
if req.method in ('PUT', 'DELETE'):
slow = self.slow - trans_time
if slow > 0:
sleep(slow)
return res(env, start_response)
def PUT(self, req):
"""HTTP PUT request handler."""
if req.body:
return jresponse('-1','param error',req,400)
if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH:
respbody = 'Container name length of %d longer than %d' % \
(len(self.container_name), MAX_CONTAINER_NAME_LENGTH)
return jresponse('-1', respbody, req,400)
account_partition, accounts = \
self.account_info(self.account_name,
autocreate=self.app.account_autocreate)
if not accounts:
return jresponse('-1', 'not found', req,404)
container_partition, containers = self.app.container_ring.get_nodes(self.account_name, self.container_name)
headers = []
for account in accounts:
nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Account-Host': '%(ip)s:%(port)s' % account,
'X-Account-Partition': account_partition,
'X-Account-Device': self.account_name,
'Connection': 'close'}
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.container_ring,
container_partition, 'PUT', req.path_info, headers)
return resp
def __call__(self, env, start_response):
req = Request(env)
if 'd' != req.headers.get('X-Ftype'):
return self.app(env,start_response)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = jresponse('-1', 'invalid UTF8', req,412)
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = jresponse('-1', 'method not allowed', req,405)
else:
res = method(req)
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = jresponse('-1', 'internal server error', req,500)
return res(env, start_response)
def __call__(self, env, start_response):
start_time = time.time()
req = Request(env)
self.logger.txn_id = req.headers.get('x-trans-id', None)
if not check_utf8(req.path_info):
res = jresponse('-1','Invalid UTF8',req,412)
else:
try:
# disallow methods which have not been marked 'public'
try:
method = getattr(self, req.method)
getattr(method, 'publicly_accessible')
except AttributeError:
res = jresponse('-1', 'method not allowed', req,405)
else:
res = method(req)
# if req.method == 'PUT':
# print 'path: '+req.path + ' status: '+str(res.status_int) + ' msg: '+res.body
except (Exception, Timeout):
self.logger.exception(_('ERROR __call__ error with %(method)s'
' %(path)s '), {'method': req.method, 'path': req.path})
res = jresponse('-1', 'InternalServerError', req,500)
return res(env, start_response)
def PUT(self, req):
"""Handle HTTP PUT request."""
if req.body:
return jresponse('-1', 'param error', req,400)
start_time = time.time()
try:
drive, part, account, container, obj = split_path(
unquote(req.path), 4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', 'bad request', req,400)
def MOVE(self,req):
if 'ftype' not in req.GET:
return jresponse('-1', 'param error', req,404)
(container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if not containers:
return jresponse('-1', 'not found', req,404)
object_partition, object_nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name)
headers = []
req.GET['ftype'] = 'f'
for container in containers:
nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Container-Host': '%(ip)s:%(port)s' % container,
'X-Container-Partition': container_partition,
'X-Container-Device': container['device'],
'x-move-dst':req.headers['Destination'],
'x-ftype':req.GET['ftype'],
'x-overwrite':req.GET.get('overwrite','false'),
'Connection': 'close'}
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.object_ring,
object_partition, 'MOVE', req.path_info, headers)
if False and object_versions:
# this is a version manifest and needs to be handled differently
lcontainer = object_versions.split('/')[0]
# prefix_len = '%03x' % len(self.object_name)
lprefix = self.object_name + '/'
last_item = None
try:
for last_item in self._listing_iter(lcontainer, lprefix,
req.environ):
pass
except ListingIterNotFound:
# no worries, last_item is None
pass
except ListingIterNotAuthorized, err:
return err.aresp
except ListingIterError:
return jresponse('-1','ServerERROR',req,500)
def DELETE(self, req):
"""HTTP DELETE request handler."""
account_partition, accounts = self.account_info(self.account_name,autocreate=False)
account = accounts[0]
(container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name)
if not containers:
return jresponse('-1', 'not found', req,404)
partition, nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name)
headers = []
for container in containers:
nheaders = dict(req.headers.iteritems())
nheaders['X-Timestamp']= normalize_timestamp(time.time())
nheaders['Connection'] = 'close'
nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
nheaders['X-Container-Partition'] = container_partition
nheaders['X-Container-Device'] = container['device']
nheaders['X-Account-Host'] = '%(ip)s:%(port)s' % account
nheaders['X-Account-Partition'] = account_partition
nheaders['X-Account-Device'] = self.account_name
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.object_ring,
partition, 'DELETE_RECYCLE', req.path_info, headers)
if object_versions and req.GET.get('cover') == 'true':
# this is a version manifest and needs to be handled differently
lcontainer = object_versions.split('/')[0]
# prefix_len = '%03x' % len(self.object_name)
lprefix = self.object_name + '/'
last_item = None
try:
for last_item in self._listing_iter(lcontainer, lprefix,
req.environ):
pass
except ListingIterNotFound:
# no worries, last_item is None
pass
except ListingIterNotAuthorized, err:
return err.aresp
except ListingIterError:
return jresponse('-1','ServerERROR',req,500)
def handle_register(self, req):
rdatas = {'failed_files':[],'success_count':0,'not_found_count':0}
req.accept = 'application/json'
out_content_type = 'application/json'
self.handle_normal(req,rdatas)
self.handle_quota(req,rdatas)
self.handle_normal_versions(req,rdatas)
self.handle_normal_metadata(req,rdatas)
self.handle_segments(req,rdatas)
self.handle_recycle(req,rdatas)
self.handle_recycle_meta(req,rdatas)
self.handle_recycle_user(req,rdatas)
self.handle_private(req,rdatas)
self.handle_private_versions(req,rdatas)
self.handle_private_metadata(req,rdatas)
self.handle_backup(req,rdatas)
self.handle_backup_versions(req,rdatas)
self.handle_backup_metadata(req,rdatas)
resp_body = get_response_body(
out_content_type,
{'Number successed': rdatas['success_count'],
'Number failed': rdatas['not_found_count']},
rdatas['failed_files'])
if (rdatas['success_count'] or rdatas['not_found_count']) and not rdatas['failed_files']:
return jresponse('0','',req,200,param=resp_body)
if rdatas['failed_files']:
return jresponse('-1',json.dumps(resp_body), req,400)
return jresponse('-1', 'Invalid userinit register', req, 400)
def PUT(self, req):
(container_partition, containers,_) = self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if not containers:
return jresponse('-1', 'not found', req,404)
direr_partition, direr_nodes = self.app.direr_ring.get_nodes(self.account_name, self.container_name, self.direr_name)
headers = []
for container in containers:
nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Container-Host': '%(ip)s:%(port)s' % container,
'X-Container-Partition': container_partition,
'X-Container-Device': container['device'],
'x-ftype':req.GET['ftype'],
'Connection': 'close'}
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.direr_ring,
direr_partition, 'PUT', req.path_info, headers)
return resp
def best_response(self, req, statuses, reasons, bodies, server_type,
etag=None,jsondata=None):
"""
:param req: webob.Request object
:param statuses: list of statuses returned
:param reasons: list of reasons for each status
:param bodies: bodies of each response
:param server_type: type of server the responses came from
:param etag: etag
:returns: webob.Response object with the correct status, body, etc. set
"""
resp = Response(request=req)
if len(statuses):
for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST):
hstatuses = \
[s for s in statuses if hundred <= s < hundred + 100]
if len(hstatuses) > len(statuses) / 2:
status = max(hstatuses)
status_index = statuses.index(status)
resp.status = '%s %s' % (status, reasons[status_index])
resp.body = bodies[status_index]
return resp
resp.status = '503 Internal Server Error'
return jresponse('-1', 'internal server error', req,503)
def DELETE_RECYCLE(self, req):
try:
device, partition, account, src_container, src_obj = split_path(
unquote(req.path), 4, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return jresponse('-1', 'bad request', req,400)
def META(self, req):
"""Handler for HTTP GET/HEAD requests."""
partition, nodes = self.app.account_ring.get_nodes(self.account_name)
shuffle(nodes)
resp = self.META_base(req, _('Account'), partition, nodes,
req.path_info.rstrip('/'), len(nodes))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
respbody = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return jresponse('-1', respbody, req,400)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
resp = self.make_requests(self.account_name,
Request.blank('/v1/' + self.account_name),
self.app.account_ring, partition, 'PUT',
'/' + self.account_name, [headers] * len(nodes))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
self.account_name)
return resp
resp = self.META_base(req, _('Account'), partition, nodes,
req.path_info.rstrip('/'), len(nodes))
return resp
def gluster_check_object_creation(req, object_name):
"""
Check to ensure that everything is alright about an object to be created.
Monkey patches swift.common.constraints.check_object_creation, invoking
the original, and then adding an additional check for individual object
name components.
:param req: HTTP request object
:param object_name: name of object to be created
:raises HTTPRequestEntityTooLarge: the object is too large
:raises HTTPLengthRequered: missing content-length header and not
a chunked request
:raises HTTPBadRequest: missing or bad content-type header, or
bad metadata
"""
ret = __check_object_creation(req, object_name)
if ret is None:
for obj in object_name.split('/'):
reason = validate_obj_name_component(obj)
if reason:
bdy = 'Invalid object name "%s", component "%s" %s' \
% (object_name, obj, reason)
ret = jresponse('-1',bdy,req,400)
return ret
def POST(self, req):
"""HTTP POST request handler."""
account_partition, accounts = self.app.account_ring.get_nodes(self.account_name)
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
self.transfer_headers(req.headers, headers)
resp = self.make_requests(self.account_name,req, self.app.account_ring,
account_partition, 'POST', req.path_info,
[headers] * len(accounts))
if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate:
if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH:
respbody = 'Account name length of %d longer than %d' % \
(len(self.account_name), MAX_ACCOUNT_NAME_LENGTH)
return jresponse('-1', respbody, req,400)
resp = self.make_requests(self.account_name,
Request.blank('/v1/' + self.account_name),
self.app.account_ring, account_partition, 'PUT',
'/' + self.account_name, [headers] * len(accounts))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
self.account_name)
return resp
return resp
def GET(self, req):
try:
drive, part, account, container, direr = split_path(
unquote(req.path), 4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', 'bad request', req,400)
def POST(self, req):
error_response = check_metadata(req, 'object')
if error_response:
return error_response
container_partition, containers,_ = self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if not containers:
return jresponse('-1', 'not found', req,404)
partition, nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name)
req.headers['X-Timestamp'] = normalize_timestamp(time.time())
headers = []
for container in containers:
nheaders = dict(req.headers.iteritems())
nheaders['Connection'] = 'close'
nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container
nheaders['X-Container-Partition'] = container_partition
nheaders['X-Container-Device'] = container['device']
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.object_ring, partition,
'POST', req.path_info, headers)
return resp
def DELETE_RECYCLE(self, req):
try:
drive, part, account, src_container, src_direr = split_path(
unquote(req.path), 4, 5, True)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', str(err), req,400)
def MOVE_VERSION(self,req):
(container_partition, containers,_) = self.container_info(self.account_name, self.container_name,
account_autocreate=self.app.account_autocreate)
if not containers:
return jresponse('-1', 'not found', req,404)
object_partition, object_nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name)
headers = []
req.GET['ftype'] = 'f'
for container in containers:
nheaders = {'X-Timestamp': normalize_timestamp(time.time()),
'x-trans-id': self.trans_id,
'X-Container-Host': '%(ip)s:%(port)s' % container,
'X-Container-Partition': container_partition,
'X-Container-Device': container['device'],
'x-move-dst':req.headers['Destination'],
'x-ftype':req.GET['ftype'],
'x-fhr-dir':'True',
'Connection': 'close'}
self.transfer_headers(req.headers, nheaders)
headers.append(nheaders)
resp = self.make_requests(self.account_name,req, self.app.object_ring,
object_partition, 'MOVE', req.path_info, headers)
return resp
def verify_request(self,req,env):
try:
version, account, container, obj = split_path(req.path_info,
minsegs=1, maxsegs=4, rest_with_last=True)
except ValueError:
self.logger.increment('errors')
return jresponse('-1','not found',req,404)
token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
verify_flag = False
if token :
user_info = self.get_cache_user_info(env, token)
if user_info:
tenant = 'AUTH_' + user_info.replace('@','').replace('.','')
if account != tenant:
self.logger.increment('unauthorized')
verify_flag = False
verify_flag = True
else:
self.logger.increment('unauthorized')
verify_flag = False
else:
self.logger.increment('unauthorized')
verify_flag = False
oauth_data_list = json.dumps({'verify_flag':str(verify_flag).lower()})
return Response(body=oauth_data_list,request=req)
def DELETE(self, req):
"""Handle HTTP DELETE request."""
try:
drive, part, account = split_path(unquote(req.path), 3)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', 'bad request',req,400)
def PUT(self, request):
try:
device, partition, account, src_container, src_link = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return jresponse('-1', 'bad request', request,400)
def HEAD(self, req):
"""Handle HTTP HEAD request."""
try:
drive, part, account, container = split_path(unquote(req.path),
3, 4)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', 'bad request', req,400)
def POST(self, req):
"""Handle HTTP POST request."""
start_time = time.time()
try:
drive, part, account, container = split_path(unquote(req.path), 4)
validate_device_partition(drive, part)
except ValueError, err:
return jresponse('-1', 'bad request', req,400)
def DELETE(self, request):
"""Handle HTTP DELETE requests for the Swift Object Server."""
start_time = time.time()
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, e:
return jresponse('-1', 'bad request', request,400)
def GET(self, request):
# request is global , can not be modify
# response can be modify
start_time = time.time()
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return jresponse('-1', 'bad request', request,400)
def check_metadata(req, target_type):
"""
Check metadata sent in the request headers.
:param req: request object
:param target_type: str: one of: object, container, or account: indicates
which type the target storage for the metadata is
:raises HTTPBadRequest: bad metadata
"""
prefix = 'x-%s' % (target_type)
meta_count = 0
meta_size = 0
for key, value in req.headers.iteritems():
if not key.lower().startswith(prefix):
continue
if not key:
respbody='Metadata name cannot be empty'
return jresponse('-1', respbody, req,400)
meta_count += 1
meta_size += len(key) + len(value)
if len(key) > MAX_META_NAME_LENGTH:
respbody='Metadata name too long; max %d' % MAX_META_NAME_LENGTH
return jresponse('-1', respbody, req,400)
elif len(value) > MAX_META_VALUE_LENGTH:
respbody='Metadata value too long; max %d' % MAX_META_VALUE_LENGTH
return jresponse('-1', respbody, req,400)
elif meta_count > MAX_META_COUNT:
respbody='Too many metadata items; max %d' % MAX_META_COUNT
return jresponse('-1', respbody, req,400)
elif meta_size > MAX_META_OVERALL_SIZE:
respbody='Total metadata too large; max %d' % MAX_META_OVERALL_SIZE
return jresponse('-1', respbody, req,400)
return None
